tornado中TCPClient、IOStream、AsyncHTTPClient、CurlAsyncHTTPClient和asyncio中的iostreams 这几个异步client的比较

最新推荐文章于 2020-05-08 23:17:12 发布

ZRXSLYG

最新推荐文章于 2020-05-08 23:17:12 发布

阅读量1.1k

点赞数

分类专栏： python

本文链接：https://blog.csdn.net/ZRXSLYG/article/details/88261635

版权

python 专栏收录该内容

13 篇文章 0 订阅

订阅专栏

tornado中TCPClient

#!/usr/bin/python3
import tornado.web
from tornado import httpserver
from tornado import ioloop
from tornado.tcpclient import TCPClient
import socket
import logging
import ssl

TimeHandler = logging.handlers.TimedRotatingFileHandler(filename = './log/tornado',when='H')
TimeHandler.suffix = '%Y%m%d-%H%M.log'

logging.basicConfig(
	#filename = para['log_file'],
	format='%(asctime)s - %(levelname)s - %(message)s',
	#filemode = 'a',
	level = logging.DEBUG,
	handlers = [TimeHandler]
)
#logging.basicConfig(filename = 'tornado.log',format = '%(asctime)s - %(levelname)s -%(message)s',filemode = 'a' ,level = logging.DEBUG)

client = TCPClient()

class IndexHandler(tornado.web.RequestHandler):
	async def get(self,*args,**kwargs):
		client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
		client_context.check_hostname = False
		client_context.verify_mode = ssl.CERT_NONE

		stream = await client.connect('www.baidu.com',443,ssl_options = client_context,timeout = 10.0)
			
		content = 'GET / HTTP/1.0\r\nAccept: */*\r\nAccept-Encoding: gzip, deflate, br\r\nAccept-Language: zh-Hans-CN, zh-Hans; q=0.8, en-US; q=0.5, en; q=0.3\r\nConnection: close\r\nHost: www.baidu.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134\r\n\r\n'
		await stream.write(content.encode())
			
		headers = {}
		header_data = b''
		body = b''
		header_data = await stream.read_until(b'\r\n\r\n')	
		for lines in header_data.split(b'\r\n'):
			parts = lines.split(b':')
			if len(parts) == 2:
				headers[parts[0].strip()] = parts[1].strip()
		
		if headers.get(b'Content-Length'):
			body = await stream.read_bytes(int(headers[b'Content-Length']))
		
		stream.close()
		if header_data != b'':
			logging.debug(header_data)
		if body != b'':
			logging.debug(body)
		
		self.write('hello world')
		self.finish()
		pass
			

def main():
	sockets = tornado.netutil.bind_sockets(7555,family = socket.AF_INET)
	app = tornado.web.Application([(r"/", IndexHandler),])
	httpd = httpserver.HTTPServer(app)
	httpd.add_sockets(sockets)
	ioloop.IOLoop.current().start()
	pass

if __name__== '__main__':
	main()

tornado中IOStream（TCPClient用的就是IOStream）

#!/usr/bin/python3
import tornado.web
from tornado import httpserver
from tornado import ioloop
from tornado.tcpclient import TCPClient
from tornado.iostream import IOStream
import socket
import logging
import ssl

TimeHandler = logging.handlers.TimedRotatingFileHandler(filename = './log/tornado',when='H')
TimeHandler.suffix = '%Y%m%d-%H%M.log'

logging.basicConfig(
	#filename = para['log_file'],
	format='%(asctime)s - %(levelname)s - %(message)s',
	#filemode = 'a',
	level = logging.DEBUG,
	handlers = [TimeHandler]
)
#logging.basicConfig(filename = 'tornado.log',format = '%(asctime)s - %(levelname)s -%(message)s',filemode = 'a' ,level = logging.DEBUG)




class IndexHandler(tornado.web.RequestHandler):
	async def get(self,*args,**kwargs):
		s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
			
		client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
		client_context.check_hostname = False
		client_context.verify_mode = ssl.CERT_NONE
				
		stream = IOStream(s)	
		await stream.connect(('www.baidu.com',443))	
		stream_ssl = await stream.start_tls(False,ssl_options = client_context)#ssl_options = dict(cert_reqs=ssl.CERT_NONE)
		await stream_ssl.wait_for_handshake()
			
		await stream_ssl.write(b'GET / HTTP/1.0\r\nAccept: */*\r\nAccept-Encoding: gzip, deflate, br\r\nAccept-Language: zh-Hans-CN, zh-Hans; q=0.8, en-US; q=0.5, en; q=0.3\r\nConnection: close\r\nHost: www.baidu.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134\r\n\r\n')
		
		headers = {}
		header_data = b''
		body = b''
		header_data = await stream_ssl.read_until(b'\r\n\r\n')	
		for lines in header_data.split(b'\r\n'):
			parts = lines.split(b':')
			if len(parts) == 2:
				headers[parts[0].strip()] = parts[1].strip()
		if headers.get(b'Content-Length'):
			body = await stream_ssl.read_bytes(int(headers[b'Content-Length']))
		
		stream_ssl.close()
		
		if header_data != b'':
			logging.debug(header_data)
		if body != b'':
			logging.debug(body)
	
		self.write('hello world')
		self.finish()
		pass

def main():
	sockets = tornado.netutil.bind_sockets(7555,family = socket.AF_INET)
	app = tornado.web.Application([(r"/", IndexHandler),])#一个元组的列表，元组的第一个元素，是路径，采用正则表达式书写方式，第二个元素是处理方式
	httpd = httpserver.HTTPServer(app)#,ssl_options = {'ssl_version':ssl.PROTOCOL_TLSv1_2}#'ca_certs':CA_path,'certfile':cert_path,'keyfile':key_path,
	httpd.add_sockets(sockets)
	ioloop.IOLoop.current().start()
	pass

if __name__== '__main__':
	main()

tornado中的AsyncHTTPClient

#!/usr/bin/python3
import tornado.web
from tornado import httpserver
from tornado import ioloop
from tornado.httpclient import AsyncHTTPClient
import socket
import logging
import ssl

TimeHandler = logging.handlers.TimedRotatingFileHandler(filename = './log/tornado',when='H')
TimeHandler.suffix = '%Y%m%d-%H%M.log'

logging.basicConfig(
	#filename = para['log_file'],
	format='%(asctime)s - %(levelname)s - %(message)s',
	#filemode = 'a',
	level = logging.DEBUG,
	handlers = [TimeHandler]
)

#logging.basicConfig(filename = 'tornado.log',format = '%(asctime)s - %(levelname)s -%(message)s',filemode = 'a' ,level = logging.DEBUG)

client = AsyncHTTPClient()

#出现以下的http返回的时候，需要伪装成浏览器
#<html>\r\n<head>\r\n\t<script>\r\n\t\tlocation.replace(location.href.replace("https://","http://"));\r\n\t</script>\r\n</head>\r\n<body>\r\n\t<noscript><meta http-equiv="refresh" content="0;url=http://www.baidu.com/"></noscript>\r\n</body>\r\n</html>
#在http头中设置User-Agent，可伪装成浏览器
#User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134
class IndexHandler(tornado.web.RequestHandler):
	async def get(self,*args,**kwargs):		
		response = await client.fetch('https://www.baidu.com:443/',headers = {'Accept':'*/*','Accept-Encoding':'gzip, deflate, br','Accept-Language':'zh-Hans-CN, zh-Hans; q=0.8, en-US; q=0.5, en; q=0.3','Host':'www.baidu.com','Connection':'close','User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134'})
		
		logging.debug(response.code)
		logging.debug(response.headers)
		logging.debug(response.body)

		self.write('Hello World')
		self.finish()
		pass	

def main():
	sockets = tornado.netutil.bind_sockets(7555,family = socket.AF_INET)
	app = tornado.web.Application([(r"/", IndexHandler),])
	httpd = httpserver.HTTPServer(app)
	httpd.add_sockets(sockets)
	ioloop.IOLoop.current().start()
	pass

if __name__== '__main__':
	main()

tornado中的CurlAsyncHTTPClient

#!/usr/bin/python3
import tornado.web
from tornado import httpserver
from tornado import ioloop
from tornado.curl_httpclient import CurlAsyncHTTPClient
import socket
import logging
import ssl

TimeHandler = logging.handlers.TimedRotatingFileHandler(filename = './log/tornado',when='H')
TimeHandler.suffix = '%Y%m%d-%H%M.log'

logging.basicConfig(
	#filename = para['log_file'],
	format='%(asctime)s - %(levelname)s - %(message)s',
	#filemode = 'a',
	level = logging.DEBUG,
	handlers = [TimeHandler]
)

#logging.basicConfig(filename = 'tornado.log',format = '%(asctime)s - %(levelname)s -%(message)s',filemode = 'a' ,level = logging.DEBUG)

client = CurlAsyncHTTPClient(max_clients = 10)

#出现以下的http返回的时候，需要伪装成浏览器
#<html>\r\n<head>\r\n\t<script>\r\n\t\tlocation.replace(location.href.replace("https://","http://"));\r\n\t</script>\r\n</head>\r\n<body>\r\n\t<noscript><meta http-equiv="refresh" content="0;url=http://www.baidu.com/"></noscript>\r\n</body>\r\n</html>
#在http头中设置User-Agent，可伪装成浏览器
#User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134
class IndexHandler(tornado.web.RequestHandler):
	async def get(self,*args,**kwargs):		
		response = await client.fetch('https://www.baidu.com:443/',headers = {'Accept':'*/*','Accept-Encoding':'gzip, deflate, br','Accept-Language':'zh-Hans-CN, zh-Hans; q=0.8, en-US; q=0.5, en; q=0.3','Host':'www.baidu.com','Connection':'close','User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134'})
		
		logging.debug(response.code)
		logging.debug(response.headers)
		logging.debug(response.body)

		self.write('Hello World')
		self.finish()
		pass
			
def main():
	sockets = tornado.netutil.bind_sockets(7555,family = socket.AF_INET)
	app = tornado.web.Application([(r"/", IndexHandler),])
	httpd = httpserver.HTTPServer(app)
	httpd.add_sockets(sockets)
	ioloop.IOLoop.current().start()
	pass

if __name__== '__main__':
	main()

asyncio中的iostreams

#!/usr/bin/python3
import tornado.web
from tornado import httpserver
from tornado import ioloop
import socket
import logging
import ssl
import asyncio
import struct

TimeHandler = logging.handlers.TimedRotatingFileHandler(filename = './log/tornado',when='H')
TimeHandler.suffix = '%Y%m%d-%H%M.log'

logging.basicConfig(
	#filename = para['log_file'],
	format='%(asctime)s - %(levelname)s - %(message)s',
	#filemode = 'a',
	level = logging.DEBUG,
	handlers = [TimeHandler]
)
#logging.basicConfig(filename = 'tornado.log',format = '%(asctime)s - %(levelname)s -%(message)s',filemode = 'a' ,level = logging.DEBUG)



class IndexHandler(tornado.web.RequestHandler):
	async def get(self,*args,**kwargs):
		s = socket.socket(socket.AF_INET,socket.SOCK_STREAM,socket.IPPROTO_TCP)
		#s.setsockopt(socket.SOL_SOCKET,socket.SO_RCVTIMEO,struct.pack('ll',3,0))
		#s.setsockopt(socket.SOL_SOCKET,socket.SO_SNDTIMEO,struct.pack('ll',3,0))
		#s.setsockopt(socket.SOL_SOCKET,socket.SO_KEEPALIVE,1)
		#s.setsockopt(socket.IPPROTO_TCP,socket.TCP_KEEPIDLE,3)
		#s.setsockopt(socket.IPPROTO_TCP,socket.TCP_KEEPINTVL,1)
		#s.setsockopt(socket.IPPROTO_TCP,socket.TCP_KEEPCNT,3)
		
		client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
		client_context.check_hostname = False
		client_context.verify_mode = ssl.CERT_NONE
		
		s.connect(('www.baidu.com',443))
		#s.setblocking(False)
		reader,writer = await asyncio.open_connection(ssl = client_context,ssl_handshake_timeout = 3.0,sock = s,server_hostname = 'www.baidu.com')#ssl = True
		
		writer.write(b'GET / HTTP/1.0\r\nAccept: */*\r\nAccept-Encoding: gzip, deflate, br\r\nAccept-Language: zh-Hans-CN, zh-Hans; q=0.8, en-US; q=0.5, en; q=0.3\r\nConnection: close\r\nHost: www.baidu.com\r\nUser-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134\r\n\r\n')
		await writer.drain()
			
		headers = {}
		header_data = b''
		body = b''
		header_data = await reader.readuntil(b'\r\n\r\n')	
		for lines in header_data.split(b'\r\n'):
			parts = lines.split(b':')
			if len(parts) == 2:
				headers[parts[0].strip()] = parts[1].strip()
		if headers.get(b'Content-Length'):
			body = await reader.readexactly(int(headers[b'Content-Length']))
			
		if not writer.is_closing():
			writer.close()
		
		if header_data != b'':
			logging.debug(header_data)
		if body != b'':
			logging.debug(body)
		
		self.write('Hello World')
		self.finish()
		pass

def main():
	sockets = tornado.netutil.bind_sockets(7555,family = socket.AF_INET)
	app = tornado.web.Application([(r"/", IndexHandler),])
	httpd = httpserver.HTTPServer(app)
	httpd.add_sockets(sockets)
	ioloop.IOLoop.current().start()
	pass

if __name__== '__main__':
	main()

测试1：用wrk进行压力测试，检测内存占用情况

wrk -t 2 -c 200 -d 20s http://localhost:7555/

#Usage: wrk <options> <url>                            
#  Options:                                            
#    -c, --connections <N>  Connections to keep open   
#    -d, --duration    <T>  Duration of test           
#    -t, --threads     <N>  Number of threads to use   
#                                                      
#    -s, --script      <S>  Load Lua script file       
#    -H, --header      <H>  Add header to request      
#        --latency          Print latency statistics   
#        --timeout     <T>  Socket/request timeout     
#    -v, --version          Print version details      
#                                                      
#  Numeric arguments may include a SI unit (1k, 1M, 1G)
#  Time arguments may include a time unit (2s, 2m, 2h)

这几个异步client都连接到https，用wrk进行压力测试，会发现，tornado中的TCPClient、IOStream和asyncio的iostreams的内存占用会相当的庞大。

测试2：每秒一个curl，观察curl什么时候会停住

#!/usr/bin/bash
while true
do
        curl localhost:7555/
        sleep 1
done

测试结果还未完成

未完待续。。。

ZRXSLYG

关注

0
点赞
踩
1

收藏

觉得还不错? 一键收藏
0
评论
tornado中TCPClient、IOStream、AsyncHTTPClient、CurlAsyncHTTPClient和asyncio中的iostreams 这几个异步client的比较

tornado中TCPClient#!/usr/bin/python3import tornado.webfrom tornado import httpserverfrom tornado import ioloopfrom tornado.tcpclient import TCPClientimport socketimport loggingimport sslTim...
复制链接

扫一扫