python数据源管理

 说明：主要实现python数据源连接池管理工具，可高效的操作各类数据源：Elasticsearch、Mysql、Redis等。

1、Python操作Elasticsearch

from elasticsearch6 import Elasticsearch, helpers


def singleton(cls):
    """
    单例模式
    """
    instances = {}
    def wrapper(*args, **kwargs):
        if cls not in instances:
            instances[cls] = cls(*args, **kwargs)
        return instances[cls]
    return wrapper


@singleton
class ElasticSearchUtil:
    ''' 
    初始化参数 
    '''
    def __init__(self):
        self.host = "127.0.0.1:9200"
        self.conn = Elasticsearch([self.host])

    def __del__(self):
        self.close()

    def checkInfo(self):
        '''
        当前ES的信息
        '''
        return self.conn.info()

    def insertDoc(self, index, type, body, id=None):
        '''
        插入一条数据,可指定Id,若不指定,ES会自动生成
        '''
        return self.conn.index(index=index, doc_type=type, body=body, id=id)

    def insertData(self, index, type, data):
        '''
        批量插入
        '''
        dataList = data.to_dict(orient='records')
        insertInfoList = [{"index": {}} for i in range(len(dataList))]
        temp = [dict] * (len(dataList) * 2)
        temp[::2] = insertInfoList
        temp[1::2] = dataList
        try:
            return self.conn.bulk(index=index, doc_type=type, body=temp)
        except Exception as e:
            return str(e)

    def deleteDocById(self, index, type, id):
        '''
        删除index、type、id对应的数据
        '''
        return self.conn.delete(index=index, doc_type=type, id=id)

    def deleteDocByQuery(self, index, query, type=None):
        '''
        删除idnex条件下query的所有数据
        '''
        return self.conn.delete_by_query(index=index, body=query, doc_type=type)

    def deleteAllDocByIndex(self, index, type=None):
        '''
        删除index下的所有数据
        '''
        try:
            query = {'query': {'match_all': {}}}
            return self.conn.delete_by_query(index=index, body=query, doc_type=type)
        except Exception as e:
            return str(e) + ' -> ' + index

    def searchDoc(self, index=None, type=None, body=None):
        '''
        查找index下所有的数据
        '''
        return self.conn.search(index=index, doc_type=type, body=body, ignore_unavailable=True, timeout='10s')

    def queryDocScroll(self,_index, _type, body):
        '''
        通过游标查询数据
        ''' 
       scanResp = helpers.scan(self.conn, body, scroll="1m", index=_index, doc_type=_type, timeout="10m")                 
       return scanResp 

    def getDocById(self, index, type, id):
        '''
        获取index、type、id对应的数据
        '''
        return self.conn.get(index=index, doc_type=type, id=id)

    def updateDocById(self, index, type, id, body=None):
        '''
        更新index、type、id所对应的数据
        '''
        return self.conn.update(index=index, doc_type=type, id=id, body=body)

    def close(self):
        ''' 
        关闭资源
        '''
     if self.conn is not None:
        try:
            self.conn.close()
        except Exception as e:
            pass
        finally:
            self.conn = None

 2、Python操作Mysql

import pymysql


def singleton(cls):
    """
    单例模式
    """
    instances = {}
    def wrapper(*args, **kwargs):
        if cls not in instances:
            instances[cls] = cls(*args, **kwargs)
        return instances[cls]
    return wrapper


@singleton
class MysqlHelper:
    '''
    python操作mysql的增删改查的封装
    '''

    def __init__(self, port=3306, charset='utf8'):
        '''
        初始化参数
        '''
        self.host = '127.0.0.1'
        self.port = port
        self.database = 'database'
        self.user = "user"
        self.password = "password"
        self.charset = charset

    def getConnect(self):
        '''
        获取连接对象和执行对象
        :return: 游标
        '''
        self.conn = pymysql.connect(host=self.host,
                                    user=self.user,
                                    password=self.password,
                                    database=self.database,
                                    port=self.port,
                                    charset=self.charset)
        self.cur = self.conn.cursor()

    def fetchone(self, sql, params=None):
       '''
       根据sql和参数获取一行数据
       :param sql:          sql
       :param params:       sql对象的参数元组，默认值为None
       :return:             查询的一行数据
       '''
        dataOne = None
        try:
            count = self.cur.execute(sql, params)
            if count != 0:
                dataOne = self.cur.fetchone()
        except Exception as ex:
            print(ex)
        finally:
            self.close()
        return dataOne

    def fetchall(self, sql, params=None):
       '''
       根据sql和参数获取一行数据
       :param sql:          sql
       :param params:       sql对象的参数列表，默认值为None
       :return:             查询的一行数据
       '''
        dataall = None
        try:
            count = self.cur.execute(sql, params)
            if count != 0:
                dataall = self.cur.fetchall()
        except Exception as ex:
            print(ex)
        finally:
            self.close()
        return dataall

    def __item(self, sql, params=None):
        '''
        增删改
        :param sql:           sql
        :param params:        sql对象的参数列表，默认值为None
        :return:              行数
        '''
        count = 0
        try:
            count = self.cur.execute(sql, params)
            self.conn.commit()
        except Exception as ex:
            print(ex)
        finally:
            self.close()
        return count

    def update(self, sql, params=None):
        '''
        修改
        :param sql:     sql
        :param params:  sql对象的参数列表，默认值为None
        :return:        行数
        '''
        return self.__item(sql, params)

    def insert(self, sql, params=None):
        '''
        新增
        :param sql:     sql
        :param params:  sql对象的参数列表，默认值为None
        :return:        行数
        '''
        return self.__item(sql, params)

    def delete(self, sql, params=None):
        '''
        删除
        :param sql:     sql
        :param params:  sql对象的参数列表，默认值为None
        :return:        行数
        '''
        return self.__item(sql, params)

    def close(self):
        '''
        关闭资源
        '''
        if self.cur != None:
            self.cur.close()
        if self.conn != None:
            self.conn.close()

 3、Python操作Redis

import redis


def singleton(cls):
    """
    单例模式
    """
    instances = {}
    def wrapper(*args, **kwargs):
        if cls not in instances:
            instances[cls] = cls(*args, **kwargs)
        return instances[cls]
    return wrapper


@singleton
class RedisHlper:

    def __init__(self):
        """
        初始化参数
        """
        self.host = "127.0.0.1"
        self.port = 6379
        self.decode_responses = True

    def getConnect(self):
        """
        返回redis操作句柄
        """
        pool = redis.ConnectionPool(host=self.host, port=self.port, decode_responses=self.decode_responses)
        operation = redis.Redis(connection_pool=pool)
        return operation