thinkphp6实现redis连接池_连接池的实现 redis例子

#-*- encoding:utf-8 -*-#import pymysql#

#conn = pymysql.connect(host="127.0.0.1", port=3306, user="py",passwd="", db="codeline",client_flag=5)##conn2 = pymysql.connect(host="127.0.0.1", port=3306, user="py",## passwd="", db="codeline")#conn.ping(True)##conn2.ping(True)#while True:#print (1)#cursor = conn.cursor()#SQL = ("SELECT username FROM logint WHERE username='%s'" % username)#print(username)#cursor.execute(SQL)#user = cursor.fetchone()#conn.commit()

#Redis是StrictRedis的子类

classRedis(StrictRedis):"""Provides backwards compatibility with older versions of redis-py that

changed arguments to some commands to be more Pythonic, sane, or by

accident."""

#StrictRedis基类

classStrictRedis(object):"""Implementation of the Redis protocol.

This abstract class provides a Python interface to all Redis commands

and an implementation of the Redis protocol.

Connection and Pipeline derive from this, implementing how

the commands are sent and received to the Redis server"""

def __init__(self, host='localhost', port=6379,

db=0, password=None, socket_timeout=None,

socket_connect_timeout=None,

socket_keepalive=None, socket_keepalive_options=None,

connection_pool=None, unix_socket_path=None,

encoding='utf-8', encoding_errors='strict',

charset=None, errors=None,

decode_responses=False, retry_on_timeout=False,

ssl=False, ssl_keyfile=None, ssl_certfile=None,

ssl_cert_reqs=None, ssl_ca_certs=None):#如果connection_pool没有定义的话就尝试获取其它设置后并重新创建连接池

if notconnection_pool:if charset is notNone:

warnings.warn(DeprecationWarning('"charset" is deprecated. Use "encoding" instead'))

encoding=charsetif errors is notNone:

warnings.warn(DeprecationWarning('"errors" is deprecated. Use "encoding_errors" instead'))

encoding_errors=errors

kwargs={'db': db,'password': password,'socket_timeout': socket_timeout,'encoding': encoding,'encoding_errors': encoding_errors,'decode_responses': decode_responses,'retry_on_timeout': retry_on_timeout

}#based on input, setup appropriate connection args

if unix_socket_path is notNone:

kwargs.update({'path': unix_socket_path,'connection_class': UnixDomainSocketConnection

})else:#TCP specific options

kwargs.update({'host': host,'port': port,'socket_connect_timeout': socket_connect_timeout,'socket_keepalive': socket_keepalive,'socket_keepalive_options': socket_keepalive_options,

})ifssl:

kwargs.update({'connection_class': SSLConnection,'ssl_keyfile': ssl_keyfile,'ssl_certfile': ssl_certfile,'ssl_cert_reqs': ssl_cert_reqs,'ssl_ca_certs': ssl_ca_certs,

})

connection_pool= ConnectionPool(**kwargs)#如果没有已经创建连接池则使用已经创建的连接池如果没有连接池则默认也会创建连接池

self.connection_pool =connection_pool

self._use_lua_lock=None

self.response_callbacks= self.__class__.RESPONSE_CALLBACKS.copy()#COMMAND EXECUTION AND PROTOCOL PARSING

def execute_command(self, *args, **options):"Execute a command and return a parsed response"pool=self.connection_pool

command_name=args[0]#从连接池中获取连接执行command_name

connection = pool.get_connection(command_name, **options)try:

connection.send_command(*args)return self.parse_response(connection, command_name, **options)except(ConnectionError, TimeoutError) as e:

connection.disconnect()if not connection.retry_on_timeout andisinstance(e, TimeoutError):raiseconnection.send_command(*args)return self.parse_response(connection, command_name, **options)finally:

pool.release(connection)#创建连接池类

classConnectionPool(object):"Generic connection pool"@classmethoddef from_url(cls, url, db=None, **kwargs):"""Return a connection pool configured from the given URL.

For example::

redis://[:password]@localhost:6379/0

rediss://[:password]@localhost:6379/0

unix://[:password]@/path/to/socket.sock?db=0

Three URL schemes are supported:

redis:// creates a normal TCP socket connection

rediss:// creates a SSL wrapped TCP socket connection

unix:// creates a Unix Domain Socket connection

There are several ways to specify a database number. The parse function

will return the first specified option:

1. A ``db`` querystring option, e.g. redis://localhost?db=0

2. If using the redis:// scheme, the path argument of the url, e.g.

redis://localhost/0

3. The ``db`` argument to this function.

If none of these options are specified, db=0 is used.

Any additional querystring arguments and keyword arguments will be

passed along to the ConnectionPool class's initializer. In the case

of conflicting arguments, querystring arguments always win."""

def __init__(self, connection_class=Connection, max_connections=None,**connection_kwargs):"""Create a connection pool. If max_connections is set, then this

object raises redis.ConnectionError when the pool's limit is reached.

By default, TCP connections are created connection_class is specified.

Use redis.UnixDomainSocketConnection for unix sockets.

Any additional keyword arguments are passed to the constructor of

connection_class."""

#最大连接数默认为62

max_connections = max_connections or 2 ** 31

if not isinstance(max_connections, (int, long)) or max_connections <0:raise ValueError('"max_connections" must be a positive integer')

self.connection_class=connection_class

self.connection_kwargs=connection_kwargs

self.max_connections=max_connections#初始化线程池

self.reset()def __repr__(self):return "%s" %(

type(self).__name__,

self.connection_class.description_format%self.connection_kwargs,

)#初始化线程池

defreset(self):#获取当前的进程号,后面判断进程是否挂掉

self.pid =os.getpid()#存放已经创建的连接(计数)

self._created_connections =0#存放可用的连接对象(列表)

self._available_connections =[]#存放正在使用的连接对象(集合)

self._in_use_connections =set()#创建线程锁

self._check_lock =threading.Lock()def_checkpid(self):if self.pid !=os.getpid():

with self._check_lock:if self.pid ==os.getpid():#another thread already did the work while we waited

#on the lock.

returnself.disconnect()

self.reset()#从连接池中获取连接

def get_connection(self, command_name, *keys, **options):"Get a connection from the pool"self._checkpid()try:#从连接池中pop出一个连接对象

connection =self._available_connections.pop()exceptIndexError:#如果连接池中已经没有连接的话重新创建连接

connection =self.make_connection()#当前正在使用的连接集合中添加此连接

self._in_use_connections.add(connection)#并返回此连接对象

returnconnection#创建连接

defmake_connection(self):"Create a new connection"

#如果大于最大连接数则抛出异常

if self._created_connections >=self.max_connections:raise ConnectionError("Too many connections")#否则创建的连接数++

self._created_connections += 1

#利用创建连接类实例化一个连接

return self.connection_class(**self.connection_kwargs)#释放连接

defrelease(self, connection):"Releases the connection back to the pool"self._checkpid()if connection.pid !=self.pid:return

#并没有关闭连接而是从在使用的连接列表中删除此连接,回收连接对象

self._in_use_connections.remove(connection)#可用连接就回收了一个连接对象

self._available_connections.append(connection)#关闭连接

defdisconnect(self):"Disconnects all connections in the pool"

#关联获取所有可迭代对象获取所有连接对象

all_conns =chain(self._available_connections,

self._in_use_connections)#便历关闭所有连接

for connection inall_conns:

connection.disconnect()#创建连接类

classConnection(object):"Manages TCP communication to and from a Redis server"description_format= "Connection"

def __init__(self, host='localhost', port=6379, db=0, password=None,

socket_timeout=None, socket_connect_timeout=None,

socket_keepalive=False, socket_keepalive_options=None,

retry_on_timeout=False, encoding='utf-8',

encoding_errors='strict', decode_responses=False,

parser_class=DefaultParser, socket_read_size=65536):

self.pid=os.getpid()

self.host=host

self.port=int(port)

self.db=db

self.password=password

self.socket_timeout=socket_timeout

self.socket_connect_timeout= socket_connect_timeout orsocket_timeout

self.socket_keepalive=socket_keepalive

self.socket_keepalive_options= socket_keepalive_options or{}

self.retry_on_timeout=retry_on_timeout

self.encoding=encoding

self.encoding_errors=encoding_errors

self.decode_responses=decode_responses

self._sock=None

self._parser= parser_class(socket_read_size=socket_read_size)

self._description_args={'host': self.host,'port': self.port,'db': self.db,

}

self._connect_callbacks=[]def __repr__(self):return self.description_format %self._description_args#对象删除时调用disconnect方法关闭对象

def __del__(self):try:

self.disconnect()exceptException:pass

defregister_connect_callback(self, callback):

self._connect_callbacks.append(callback)defclear_connect_callbacks(self):

self._connect_callbacks=[]#核心的连接方法还是通过socket模块儿实现

defconnect(self):"Connects to the Redis server if not already connected"

ifself._sock:return

try:#私有方法创建连接

sock =self._connect()exceptsocket.error:

e= sys.exc_info()[1]raiseConnectionError(self._error_message(e))

self._sock=socktry:

self.on_connect()exceptRedisError:#clean up after any error in on_connect

self.disconnect()raise

#run any user callbacks. right now the only internal callback

#is for pubsub channel/pattern resubscription

for callback inself._connect_callbacks:

callback(self)#创建tcp socket

def_connect(self):"Create a TCP socket connection"

#we want to mimic what socket.create_connection does to support

#ipv4/ipv6, but we want to set options prior to calling

#socket.connect()

err =Nonefor res insocket.getaddrinfo(self.host, self.port, 0,

socket.SOCK_STREAM):

family, socktype, proto, canonname, socket_address=res

sock=Nonetry:

sock=socket.socket(family, socktype, proto)#TCP_NODELAY

sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)#TCP_KEEPALIVE

#默认使用的是短连接,设置socket_keepalive=True保持长连接

ifself.socket_keepalive:

sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE,1)for k, v initeritems(self.socket_keepalive_options):

sock.setsockopt(socket.SOL_TCP, k, v)#设置超市时间,默认不设置为阻塞模式

#set the socket_connect_timeout before we connect

sock.settimeout(self.socket_connect_timeout)#connect

sock.connect(socket_address)#set the socket_timeout now that we're connected

sock.settimeout(self.socket_timeout)returnsockexceptsocket.error as _:

err=_if sock is notNone:

sock.close()if err is notNone:raiseerrraise socket.error("socket.getaddrinfo returned an empty list")def_error_message(self, exception):#args for socket.error can either be (errno, "message")

#or just "message"

if len(exception.args) == 1:return "Error connecting to %s:%s. %s." %\

(self.host, self.port, exception.args[0])else:return "Error %s connecting to %s:%s. %s." %\

(exception.args[0], self.host, self.port, exception.args[1])defon_connect(self):"Initialize the connection, authenticate and select a database"self._parser.on_connect(self)#if a password is specified, authenticate

ifself.password:

self.send_command('AUTH', self.password)if nativestr(self.read_response()) != 'OK':raise AuthenticationError('Invalid Password')#if a database is specified, switch to it

ifself.db:

self.send_command('SELECT', self.db)if nativestr(self.read_response()) != 'OK':raise ConnectionError('Invalid Database')#关闭连接

defdisconnect(self):"Disconnects from the Redis server"self._parser.on_disconnect()if self._sock isNone:return

try:#先shutdown然后再close

self._sock.shutdown(socket.SHUT_RDWR)

self._sock.close()exceptsocket.error:passself._sock=Nonedefsend_packed_command(self, command):"Send an already packed command to the Redis server"

if notself._sock:

self.connect()try:ifisinstance(command, str):

command=[command]for item incommand:

self._sock.sendall(item)exceptsocket.timeout:

self.disconnect()raise TimeoutError("Timeout writing to socket")exceptsocket.error:

e= sys.exc_info()[1]

self.disconnect()if len(e.args) == 1:

_errno, errmsg= 'UNKNOWN', e.args[0]else:

_errno, errmsg=e.argsraise ConnectionError("Error %s while writing to socket. %s." %(_errno, errmsg))except:

self.disconnect()raise

def send_command(self, *args):"Pack and send a command to the Redis server"self.send_packed_command(self.pack_command(*args))defcan_read(self):"Poll the socket to see if there's data that can be read."sock=self._sockif notsock:

self.connect()

sock=self._sockreturn bool(select([sock], [], [], 0)[0]) orself._parser.can_read()defread_response(self):"Read the response from a previously sent command"

try:

response=self._parser.read_response()except:

self.disconnect()raise

ifisinstance(response, ResponseError):raiseresponsereturnresponsedefencode(self, value):"Return a bytestring representation of the value"

ifisinstance(value, Token):returnb(value.value)elifisinstance(value, bytes):returnvalueelifisinstance(value, (int, long)):

value=b(str(value))elifisinstance(value, float):

value=b(repr(value))elif notisinstance(value, basestring):

value=str(value)ifisinstance(value, unicode):

value=value.encode(self.encoding, self.encoding_errors)returnvaluedef pack_command(self, *args):"Pack a series of arguments into the Redis protocol"output=[]#the client might have included 1 or more literal arguments in

#the command name, e.g., 'CONFIG GET'. The Redis server expects these

#arguments to be sent separately, so split the first argument

#manually. All of these arguements get wrapped in the Token class

#to prevent them from being encoded.

command =args[0]if ' ' incommand:

args= tuple([Token(s) for s in command.split(' ')]) + args[1:]else:

args= (Token(command),) + args[1:]

buff=SYM_EMPTY.join(

(SYM_STAR, b(str(len(args))), SYM_CRLF))for arg inimap(self.encode, args):#to avoid large string mallocs, chunk the command into the

#output list if we're sending large values

if len(buff) > 6000 or len(arg) > 6000:

buff=SYM_EMPTY.join(

(buff, SYM_DOLLAR, b(str(len(arg))), SYM_CRLF))

output.append(buff)

output.append(arg)

buff=SYM_CRLFelse:

buff=SYM_EMPTY.join((buff, SYM_DOLLAR, b(str(len(arg))),

SYM_CRLF, arg, SYM_CRLF))

output.append(buff)returnoutputdefpack_commands(self, commands):"Pack multiple commands into the Redis protocol"output=[]

pieces=[]

buffer_length=0for cmd incommands:for chunk in self.pack_command(*cmd):

pieces.append(chunk)

buffer_length+=len(chunk)if buffer_length > 6000:

output.append(SYM_EMPTY.join(pieces))

buffer_length=0

pieces=[]ifpieces:

output.append(SYM_EMPTY.join(pieces))return output