python-requests-2

最新推荐文章于 2022-05-21 15:16:16 发布

baihaisheng

最新推荐文章于 2022-05-21 15:16:16 发布

阅读量944

点赞数

分类专栏：数据爬虫文章标签：自然语言处理数据挖掘

本文链接：https://blog.csdn.net/qq_41128383/article/details/106159549

版权

数据爬虫专栏收录该内容

4 篇文章 0 订阅

订阅专栏

requests.get()

import requests
response = requests.get('http://www.baidu.com')
print("打印状态码_tatus_code",response.status_code)
print("打印请求_url :",response.url)
print("打印头信息_headers:",response.headers)
print("打印cookie信息_cookies：",response.cookies)
print("请求内容_text：",response.text)
print("以字节流形式打印_content:",response.content)

打印状态码_tatus_code 200
打印请求_url : http://www.baidu.com/
打印头信息_headers: {'Cache-Control': 'private, no-cache, no-store, proxy-revalidate, no-transform', 'Connection': 'keep-
alive', 'Content-Encoding': 'gzip', 'Content-Type': 'text/html', 'Date': 'Sat, 16 May 2020 07:11:48 GMT', 'Last-Modified': 'Mon, 23 Jan 2017 13:27:57 GMT', 'Pragma': 'no-cache', 'Server': 'bfe/1.0.8.18', 'Set-Cookie': 'BDORZ=27315; max-age=86400; domain=.baidu.com; path=/', 'Transfer-Encoding': 'chunked'}打印cookie信息_cookies： <RequestsCookieJar[<Cookie BDORZ=27315 for .baidu.com/>]>

请求内容_text： <!DOCTYPE html>
<!--STATUS OK--><html> <head><meta http-equiv=content-type content=text/html;charset=utf-8><meta http-equiv=X-UA-Compatib
le content=IE=Edge><meta content=always name=referrer><link rel=stylesheet type=text/css href=http://s1.bdstatic.com/r/www/cache/bdorz/baidu.min.css><title>ç¾åº¦ä¸ä¸ï¼ä½ å°±ç¥é</title></head> <body link=#0000cc> <div id=wrapper> <div id=head> <div class=head_wrapper> <div class=s_form> <div class=s_form_wrapper> <div id=lg> <img hidefocus=true src=//www.baidu.com/img/bd_logo1.png width=270 height=129> </div> <form id=form name=f action=//www.baidu.com/s class=fm> <input type=hidden name=bdorz_come value=1> <input type=hidden name=ie value=utf-8> <input type=hidden name=f value=8> <input type=hidden name=rsv_bp value=1> <input type=hidden name=rsv_idx value=1> <input type=hidden name=tn value=baidu><span class="bg s_ipt_wr"><input id=kw name=wd class=s_ipt value maxlength=255 autocomplete=off autofocus></span><span class="bg s_btn_wr"><input type=submit id=su value=ç¾åº¦ä¸ä¸ class="bg s_btn"></span> </form> </div> </div> <div id=u1> <a href=http://news.baidu.com name=tj_trnews class=mnav>æ°é»</a> <a href=http://www.hao123.com name=tj_trhao123 class=mnav>hao123</a> <a href=http://map.baidu.com name=tj_trmap class=mnav>å°å¾</a> <a href=http://v.baidu.com name=tj_trvideo class=mnav>è§é¢</a> <a href=http://tieba.baidu.com name=tj_trtieba class=mnav>è´´å§</a> <noscript> <a href=http://www.baidu.com/bdorz/login.gif?login&amp;tpl=mn&amp;u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 name=tj_login class=lb>ç»å½</a> </noscript> <script>document.write('<a href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u='+ encodeURIComponent(window.location.href+ (window.location.search === "" ? "?" : "&")+ "bdorz_come=1")+ '" name="tj_login" class="lb">ç»å½</a>');</script> <a href=//www.baidu.com/more/ name=tj_briicon class=bri style="display: block;">æ´å¤äº§å</a> </div> </div> </div> <div id=ftCon> <div id=ftConw> <p id=lh> <a href=http://home.baidu.com>å³äºç¾åº¦</a> <a href=http://ir.baidu.com>About Baidu</a> </p> <p id=cp>&copy;2017&nbsp;Baidu&nbsp;<a href=http://www.baidu.com/duty/>ä½¿ç¨ç¾åº¦åå¿è¯»</a>&nbsp; <a href=http://jianyi.baidu.com/ class=cp-feedback>æè§åé¦</a>&nbsp;äº¬ICPè¯030173å·&nbsp; <img src=//www.baidu.com/img/gs.gif> </p> </div> </div> </div> </body> </html>


以字节流形式打印_content: b'<!DOCTYPE html>\r\n<!--STATUS OK--><html> <head><meta http-equiv=content-type content=text/htm
l;charset=utf-8><meta http-equiv=X-UA-Compatible content=IE=Edge><meta content=always name=referrer><link rel=stylesheet type=text/css href=http://s1.bdstatic.com/r/www/cache/bdorz/baidu.min.css><title>\xe7\x99\xbe\xe5\xba\xa6\xe4\xb8\x80\xe4\xb8\x8b\xef\xbc\x8c\xe4\xbd\xa0\xe5\xb0\xb1\xe7\x9f\xa5\xe9\x81\x93</title></head> <body link=#0000cc> <div id=wrapper> <div id=head> <div class=head_wrapper> <div class=s_form> <div class=s_form_wrapper> <div id=lg> <img hidefocus=true src=//www.baidu.com/img/bd_logo1.png width=270 height=129> </div> <form id=form name=f action=//www.baidu.com/s class=fm> <input type=hidden name=bdorz_come value=1> <input type=hidden name=ie value=utf-8> <input type=hidden name=f value=8> <input type=hidden name=rsv_bp value=1> <input type=hidden name=rsv_idx value=1> <input type=hidden name=tn value=baidu><span class="bg s_ipt_wr"><input id=kw name=wd class=s_ipt value maxlength=255 autocomplete=off autofocus></span><span class="bg s_btn_wr"><input type=submit id=su value=\xe7\x99\xbe\xe5\xba\xa6\xe4\xb8\x80\xe4\xb8\x8b class="bg s_btn"></span> </form> </div> </div> <div id=u1> <a href=http://news.baidu.com name=tj_trnews class=mnav>\xe6\x96\xb0\xe9\x97\xbb</a> <a href=http://www.hao123.com name=tj_trhao123 class=mnav>hao123</a> <a href=http://map.baidu.com name=tj_trmap class=mnav>\xe5\x9c\xb0\xe5\x9b\xbe</a> <a href=http://v.baidu.com name=tj_trvideo class=mnav>\xe8\xa7\x86\xe9\xa2\x91</a> <a href=http://tieba.baidu.com name=tj_trtieba class=mnav>\xe8\xb4\xb4\xe5\x90\xa7</a> <noscript> <a href=http://www.baidu.com/bdorz/login.gif?login&amp;tpl=mn&amp;u=http%3A%2F%2Fwww.baidu.com%2f%3fbdorz_come%3d1 name=tj_login class=lb>\xe7\x99\xbb\xe5\xbd\x95</a> </noscript> <script>document.write(\'<a href="http://www.baidu.com/bdorz/login.gif?login&tpl=mn&u=\'+ encodeURIComponent(window.location.href+ (window.location.search === "" ? "?" : "&")+ "bdorz_come=1")+ \'" name="tj_login" class="lb">\xe7\x99\xbb\xe5\xbd\x95</a>\');</script> <a href=//www.baidu.com/more/ name=tj_briicon class=bri style="display: block;">\xe6\x9b\xb4\xe5\xa4\x9a\xe4\xba\xa7\xe5\x93\x81</a> </div> </div> </div> <div id=ftCon> <div id=ftConw> <p id=lh> <a href=http://home.baidu.com>\xe5\x85\xb3\xe4\xba\x8e\xe7\x99\xbe\xe5\xba\xa6</a> <a href=http://ir.baidu.com>About Baidu</a> </p> <p id=cp>&copy;2017&nbsp;Baidu&nbsp;<a href=http://www.baidu.com/duty/>\xe4\xbd\xbf\xe7\x94\xa8\xe7\x99\xbe\xe5\xba\xa6\xe5\x89\x8d\xe5\xbf\x85\xe8\xaf\xbb</a>&nbsp; <a href=http://jianyi.baidu.com/ class=cp-feedback>\xe6\x84\x8f\xe8\xa7\x81\xe5\x8f\x8d\xe9\xa6\x88</a>&nbsp;\xe4\xba\xacICP\xe8\xaf\x81030173\xe5\x8f\xb7&nbsp; <img src=//www.baidu.com/img/gs.gif> </p> </div> </div> </div> </body> </html>\r\n'

不带参数GET请求

mport requests
response = requests.get('https://bj.fang.lianjia.com/')
print(response.text)

带参的GET请求

直接将参数放在url中

import requests

response = requests.get(http://httpbin.org/get?name=gemey&age=22)
print(response.text)

参数填写在dict中，发起请求时params参数指定为dict

import requests

data = {'dfsdf': 'hgkhg'}
response = requests.get('http://httpbin.org/get', params=data)
print(response.text)

请求添加头信息

import requests

header={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Saf
ari/537.36"}

 response = requests.get('http://www.baidu.com',headers=headers)

使用代理

使用requests库爬取了IP代理网站的IP与端口和类型

因为是免费的，使用的代理地址很快就失效了。

import requests
import re

def get_html(url):
    proxy = {'http': '120.25.253.234:812',
        'https' '163.125.222.244:8123'}
	header={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Saf
ari/537.36"}
    req = requests.get(url, headers=heads,proxies=proxy)
    html = req.text
    return html

证书验证设置

import requests
response = requests.get('https://bj.fang.lianjia.com/',verify=False)  #证书验证设为FALSE
print(response.status_code)

超时异常捕获

import requests
from requests.exceptions import ReadTimeout
try:
    res = requests.get('https://bj.fang.lianjia.com/', timeout=0.1)
    print(res.status_code)
except ReadTimeout:
    print(timeout)

异常处理

import requests
from requests.exceptions import ReadTimeout,HTTPError,RequestException

try:
    response = requests.get('https://bj.fang.lianjia.com/',timeout=0.5)
    print(response.status_code)
except ReadTimeout:
    print('timeout')
except HTTPError:
    print('httperror')
except RequestException:
    print('reqerror')

requests.POST()

带数据的post

```python
import requests

data = {'name':'tom','age':'22'}
response = requests.post('https://bj.fang.lianjia.com/', data=data)

带header的post

import requests
header={"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Saf
ari/537.36"}
data = {'name':'tom','age':'22'}
response = requests.post('https://bj.fang.lianjia.com/', headers=header，data=data)

带json的post

# -*- coding:utf-8 -*-
import requests
import json

host = "https://bj.lianjia.com/zufang/"
endpoint = "post"

url = ''.join([host,endpoint])
data = {"sites": [{ "name":"weibo" , "url":"www.weibo.com" }]}

r = requests.post(url,json=data)
response = r.json()

带参数的post

# -*- coding:utf-8 -*-
import requests
import json

host = "https://bj.lianjia.com/zufang/"
endpoint = "post"
url = ''.join([host,endpoint])
params = {'key1':'params1','key2':'params2'}

r = requests.post(url,params=params)
response = r.json()

普通文件上传

# -*- coding:utf-8 -*-
import requests
import json

host = "https://bj.lianjia.com/zufang/"
endpoint = "post"

url = ''.join([host,endpoint])
#普通上传
files = {
            'file':open('test.txt','rb')
        }

r = requests.post(url,files=files)
print (r.text)

定制化文件上传

# -*- coding:utf-8 -*-
import requests
import json

host = "https://bj.lianjia.com/zufang/"
endpoint = "post"

url = ''.join([host,endpoint])
#自定义文件名，文件类型、请求头
files = {
        'file':('test.png',open('test.png','rb'),'image/png')
}

r = requests.post(url,files=files)
print (r.text)heman793

多文件上传

# -*- coding:utf-8 -*-
import requests
import json

host = "https://bj.lianjia.com/zufang/"
endpoint = "post"

url = ''.join([host,endpoint])
#多文件上传
files = [
    ('file1',('test.txt',open('test.txt', 'rb'))),
    ('file2', ('test.png', open('test.png', 'rb')))
    ]

r = requests.post(url,files=files)
print (r.text)

baihaisheng

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
python-requests-2

requests.get()import requestsresponse = requests.get('http://www.baidu.com')print("打印状态码_tatus_code",response.status_code)print("打印请求_url :",response.url)print("打印头信息_headers:",response.headers)print("打印cookie信息_cookies：",response.cookies)print("请求内
复制链接

扫一扫

专栏目录