python elasticsearch bulk_Python 调用 elasticsearch 的 bulk 接口批量插入数据出现内存泄露，导致 OOM...

数据导入脚本如下

import time

import sys

from elasticsearch import Elasticsearch

from elasticsearch.helpers import bulk

reload(sys)

sys.setdefaultencoding('utf-8')

def set_mapping(es, index_name = "content_engine", doc_type_name = "en"):

my_mapping = {

"en": {

"properties": {

"a": {

"type": "string"

},

"b": {

"type": "string"

}

}

}

}

create_index = es.indices.create(index = index_name,body = my_mapping)

mapping_index = es.indices.put_mapping(index = index_name, doc_type = doc_type_name, body = my_mapping)

if create_index["acknowledged"] != True or mapping_index["acknowledged"] != True:

print "Index creation failed..."

def set_data(es, input_file, index_name = "content_engine", doc_type_name="en"):

i = 0

count = 0

ACTIONS = []

for line in open(input_file):

fields = line.replace("\r\n", "").replace("\n", "").split("----")

if len(fields) == 2:

a, b = fields

else:

continue

action = {

"_index": index_name,

"_type": doc_type_name,

"_source": {

"a": a,

"b": b,

}

}

i += 1

ACTIONS.append(action)

if (i == 500000):

success, _ = bulk(es, ACTIONS, index = index_name, raise_on_error = True)

count += success

i = 0

ACTIONS = []

success, _ = bulk(es, ACTIONS, index = index_name, raise_on_error=True)

count += success

print("insert %s lines" % count)

if __name__ == '__main__':

es = Elasticsearch(hosts=["127.0.0.1:9200"], timeout=5000)

set_mapping(es)

set_data(es,sys.argv[1])

数据大概 5 个 G 吧，机器配置虚拟机 24G 内存，刚开始无内存泄露现象，这个 Python 脚本的进程内存一直保持 1G 左右的占用，当插入 1600 ｗ，内存开始持续飙升，最后达到 22G ，导致触发 OOM 机制， Python 进程被内核 kill ，差点怀疑人生。。大家在遇到 Python 内存泄露都是怎么定位的？