ElasticSearch - （）使用Python单例模式批量写入数据

1.批量写入

• 如果有大量的数据，一次插入一条肯定效率太慢，我们可以使用elasticsearch模块导入helper，通过helper.bluk 来批量处理大量的数据。首先我们将所有的数据定义成字典形式，各字段含义如下：

from elasticsearch import Elasticsearch, helpers
es = Elasticsearch(**es_settings)
# 批量数据的列表
actions = [{
            '_index': index,  #索引
            '_type': doc_type,  #类型
            '_source': item_dict0  #对应字典内的字段和值，有多个字段
        }, 
        {'_index': index,'_type': doc_type, '_source': item_dict1},
		{'_index': index,'_type': doc_type, '_source': item_dict2},
		......
		]
# 使用es的 helpers.bluk方法
helpers.bluk(es, actions， stats_only=True,raise_on_error=False, raise_on_exception=False)

2.单例模式写入

from elasticsearch import Elasticsearch, helpers
class SingleES(object):
	_instance = None
	client = None
	actions = []
	def __init__(self):
		self.client = Elasticsearch(hosts=['10.10.10.165:9200'],timeout=60)
		self.actions_length = 50
	
	@staticmethod
    def instance():
    	# 不存在 _instance的属性
        if not SingleES._instance:
        	# 创建实例， 将实例赋值给_instance属性
            SingleES._instance = SingleES()
        return SingleES._instance
    def storage(self, item_dict):
    	action = {
            '_index': 'index',
            '_type': 'doc_type',
            '_source': item_dict
        }
        self.actions.append(action)
        if len(self.actions) >= self.actions_length :
	        try:
	            helpers.bulk(self.client, self.actions, stats_only=True,
	                         raise_on_error=False, raise_on_exception=False)
	        except:
	            logging.error("ElasticSearch bulk data fail!")
	        finally:
	            self.actions = []

3.多线程写入，使用线程锁

from elasticsearch import Elasticsearch, helpers
import threading


class SingleES(object):
	_instance = None
	client = None
	actions = []
	_instance_lock = threading.Lock()  # 线程锁
	
	def __init__(self):
		self.client = Elasticsearch(hosts=['10.10.10.165:9200'],timeout=60)
		self.actions_length = 50
	
	@staticmethod
    def instance():
    	# 不存在 _instance的属性
        if not SingleES._instance:
        	# 创建实例， 将实例赋值给_instance属性
            SingleES._instance = SingleES()
        return SingleES._instance
    def storage(self, item_dict):
    	action = {
            '_index': 'index',
            '_type': 'doc_type',
            '_source': item_dict
        }
        self.actions.append(action)
        if len(self.actions) >= self.actions_length :
        	# todo 使用线程锁
        	with SingleES._instance_lock:
		        if len(self.actions) >= self.actions_length :
			        try:
			            helpers.bulk(self.client, self.actions, stats_only=True,
			                         raise_on_error=False, raise_on_exception=False)
			        except:
			            logging.error("ElasticSearch bulk data fail!")
			        finally:
			            self.actions = []

if __name__ == '__main__':
    es_db = SingleES.instance()
    for i in range(10000):
        es_db.storage({'name': 'zhang', 'age':123})