python编写es脚本_Elasticsearch Python脚本查询常用操作

最新推荐文章于 2022-10-24 11:04:09 发布
最新推荐文章于 2022-10-24 11:04:09 发布
阅读量365 收藏
点赞数
文章标签： python编写es脚本

一、ES Python脚本查询操作 http方式

#!coding:utf-8

import json

import logging

import time

import requests

PAGE_RESULT_SCROLL_ID = 'scroll_id'

PAGE_RESULT_SCROLL_SIZE = 'scroll_size'

PAGE_RESULT_TOTAL_SIZE = 'total_size'

PAGE_RESULT_HITS = 'hits'

PAGE_RESULT_DATA = 'data'

PAGE_RESULT_CONVERT_DATA = 'convert_data'

CONVERT_DEST_KEY = 'dest_key'

CONVERT_DEFAULT_VALUE = 'default_value'

current_time = time.strftime("%Y-%m-%d-%H-%M", time.localtime(time.time()))

# 日志设置

log_file = "operate_es_" + current_time + ".log"

logging.FileHandler(filename=log_file, encoding='utf-8')

logging.basicConfig(filename=log_file, level=logging.INFO)

# 创建索引

def create_index(es_url, index_name, es_mapping):

logging.info("es_url:%s index_name:%s " % (es_url, index_name))

es_index_url = es_url + index_name

logging.info("es_index_url: %s" % (es_index_url))

r = requests.put(es_index_url, json.dumps(es_mapping), headers={"content-type": "application/json"})

response_json = json.loads(r.text)

if 200 == r.status_code:

if response_json['acknowledged']:

logging.info("index_name: %s 创建成功" % (index_name))

else:

logging.info("index_name: %s 创建失败" % (index_name))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

return -1

logging.info("index create done!")

return 0

# 批量创建索引

def batch_create_index_with_mapping(es_url, es_index_prefix, es_mapping, batch_num=1, start_index=0):

logging.info("es_url:" + es_url)

new_index_array = []

for i in range(start_index, batch_num):

suffix = "%03d" % i

index_name = es_index_prefix + suffix

es_index_url = es_url + index_name

logging.info("es_index_url: %s" % (es_index_url))

r = requests.put(es_index_url, json.dumps(es_mapping), headers={"content-type": "application/json"})

response_json = json.loads(r.text)

if 200 == r.status_code:

if response_json['acknowledged']:

logging.info("index_name: %s 创建成功" % (index_name))

new_index_array.append(index_name)

else:

logging.info("index_name: %s 创建失败" % (index_name))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

break

logging.info("index create done!")

logging.info("new_index_array:%s" % (new_index_array))

# 删除索引

def delete_index(es_url, delete_index):

delete_es_url = es_url + delete_index

logging.info("es_url:%s delete_index:%s start..." % (delete_es_url, delete_index))

r = requests.delete(delete_es_url, headers={"content-type": "application/json"})

if 200 == r.status_code:

response_json = json.loads(r.text)

logging.info("delete数据返回响应： %s " % r.text.encode(encoding="utf-8"))

else:

logging.info("es_url:%s delete_index: %s 删除失败" % (es_url, delete_index))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

return -1

return 0

# 根据前缀删除索引

def del_index_by_prefix(es_url, index_prefix):

delete_es_url = es_url + index_prefix + "*"

logging.info("es_url:%s start..." % (delete_es_url))

r = requests.delete(delete_es_url, headers={"content-type": "application/json"})

if 200 == r.status_code:

response_json = json.loads(r.text)

logging.info("delete数据返回响应： %s " % r.text.encode(encoding="utf-8"))

else:

logging.info("delete_es_url:%s 删除失败" % (delete_es_url))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

return -1

return 0

# 批量前缀+分桶删除索引

def batch_es_del_index(es_url, index_prefix, bucket_num, start_index=0):

logging.info("es_url:" + es_url)

delete_index_array = []

for i in range(start_index, bucket_num):

suffix = "%03d" % i

delete_index = index_prefix + suffix

ret = es_delete_index(es_url, delete_index)

if 0 == ret:

delete_index_array.append(delete_index)

else:

logging.info("delete_index:%s 失败" % (delete_index))

break

logging.info("batch_es_del_index done!")

logging.info("delete_index_array:%s" % (delete_index_array))

# 更新索引mapping

def add_properties_to_index(es_url, index_name, doc_type, es_mapping):

logging.info("es_url:" + es_url)

es_index_url = es_url + index_name + '/' + doc_type + '/_mapping'

logging.info("es_index_url: %s" % (es_index_url))

r = requests.post(es_index_url, json.dumps(es_mapping), headers={"content-type": "application/json"})

response_json = json.loads(r.text)

if 200 == r.status_code:

if response_json['acknowledged']:

logging.info("index_name: %s 更新成功" % (index_name))

else:

logging.info("index_name: %s 更新失败" % (index_name))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

return -1

logging.info("index modify done!")

return 0

# 批量更新索引mapping

def batch_add_properties_to_index(es_url, index_prefix, doc_type, es_mapping, bucket_num, start_index=0):

logging.info("es_url:" + es_url)

new_index_array = []

for i in range(start_index, bucket_num):

suffix = "%03d" % i

index_name = index_prefix + suffix

es_index_url = es_url + index_name + '/' + doc_type + '/_mapping'

logging.info("es_index_url: %s" % (es_index_url))

r = requests.post(es_index_url, json.dumps(es_mapping), headers={"content-type": "application/json"})

response_json = json.loads(r.text)

if 200 == r.status_code:

if response_json['acknowledged']:

logging.info("index_name: %s 更新成功" % (index_name))

new_index_array.append(index_name)

else:

logging.info("index_name: %s 更新失败" % (index_name))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

logging.info("index modify done new_index_array:%s" % (new_index_array))

return -1

logging.info("index modify done!")

logging.info("new_index_array:%s" % (new_index_array))

return 0

# 备份索引内容(结构+数据)

def es_reindex_with_routing(source_index, bak_index, query, routing_filed):

# 备份数据

url = es_url + "_reindex"

data = {

"source": {

"index": source_index,

"query": query

},

"dest": {

"index": bak_index

},

"script": {

"inline": "ctx._routing = ctx._source." + routing_filed,

"lang": "painless"

}

}

logging.info("source_index:%s to bak_index: %s start..." % (source_index, bak_index))

r = requests.post(url, json.dumps(data), headers={"content-type": "application/json"})

if 200 == r.status_code:

response_json = json.loads(r.text)

logging.info("备份数据返回响应： %s " % r.text.encode(encoding="utf-8"))

logging.info("source_index:%s to bak_index: %s 复制成功" % (source_index, bak_index))

logging.info("备份索引： %s , 备份数据量： %d" % (bak_index, response_json['total']))

else:

logging.info("source_index:%s bak_index: %s 备份失败" % (source_index, bak_index))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

return -1

return 0

# 备份索引

def es_reindex(source_index, bak_index, query):

# 备份数据

url = es_url + "_reindex"

data = {

"source": {

"index": source_index,

"query": query

},

"dest": {

"index": bak_index

}

}

logging.info("source_index:%s to bak_index: %s start..." % (source_index, bak_index))

r = requests.post(url, json.dumps(data), headers={"content-type": "application/json"})

if 200 == r.status_code:

response_json = json.loads(r.text)

logging.info("备份数据返回响应： %s " % r.text.encode(encoding="utf-8"))

logging.info("source_index:%s to bak_index: %s 复制成功" % (source_index, bak_index))

logging.info("备份索引： %s , 备份数据量： %d" % (bak_index, response_json['total']))

else:

logging.info("source_index:%s bak_index: %s 备份失败" % (source_index, bak_index))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

return -1

return 0

# 批量备份

def batch_es_reindex(es_url, source_index_prefix, bak_index_prefix, queryParam, bucket_num, start_index=0):

'''

:param es_url:

:param source_index_prefix:

:param bak_index_prefix:

:param queryParam:

queryParam = {

"query": {

"bool": {

"must": [

{

"match_all": {}

}

],

"must_not": [],

"should": []

}

}

}

:param bucket_num:

:param start_index:

:return:

'''

logging.info("es_url:" + es_url)

new_index_array = []

for i in range(start_index, bucket_num):

suffix = "%03d" % i

source_index = source_index_prefix + suffix

bak_index = bak_index_prefix + suffix

ret = es_reindex(source_index, bak_index, queryParam["query"])

if 0 == ret:

new_index_array.append(bak_index)

else:

logging.info("source_index:%s to bak_index: %s 复制失败" % (source_index, bak_index))

logging.info("do new_index_array:%s over" % (new_index_array))

return -1

logging.info("batch_es_reindex done!")

logging.info("new_index_array:%s" % (new_index_array))

return 0

# 批量备份加路由指定

def batch_es_reindex_with_routing(es_url, source_index_prefix, bak_index_prefix, queryParam, routing_filed, bucket_num,

start_index=0):

'''

:param es_url:

:param source_index_prefix:

:param bak_index_prefix:

:param queryParam:

queryParam = {

"query": {

"bool": {

"must": [

{

"match_all": {}

}

],

"must_not": [],

"should": []

}

}

}

:param bucket_num:

:param routing_filed:

:param start_index:

:return:

'''

logging.info("es_url:" + es_url)

new_index_array = []

for i in range(start_index, bucket_num):

suffix = "%03d" % i

source_index = source_index_prefix + suffix

bak_index = bak_index_prefix + suffix

ret = es_reindex_with_routing(source_index, bak_index, queryParam["query"], routing_filed)

if 0 == ret:

new_index_array.append(bak_index)

else:

logging.info("source_index:%s to bak_index: %s 复制失败" % (source_index, bak_index))

logging.info("do new_index_array:%s over" % (new_index_array))

return -1

logging.info("batch_es_reindex done!")

logging.info("new_index_array:%s" % (new_index_array))

return 0

# 根据业务规则创建索引

def create_index_by_business_code_rel_type_dict(es_url, es_index_prefix, es_mapping,

business_codes_rel_type_dict,

do_reindex=False,

source_index='',

routing_field=''):

'''

:param es_url:

:param es_index_prefix:

:param es_mapping:

:param business_codes_rel_type_dict:

business_codes_rel_type_dict = {"003": ["fk_tl"],"004": ["bq_sb"],"005": [ "jd_zh", "jd_zz"]}

:param do_reindex:

:param source_index:

:param routing_field:

:return:

'''

logging.info("es_url:" + es_url)

new_index_array = []

business_codes = business_codes_rel_type_dict.keys()

for business_code in business_codes:

relation_types = business_codes_rel_type_dict.get(business_code)

for relation_type in relation_types:

index_name = es_index_prefix + business_code + "_" + relation_type

es_index_url = es_url + index_name

logging.info("es_index_url: %s" % (es_index_url))

r = requests.put(es_index_url, json.dumps(es_mapping), headers={"content-type": "application/json"})

response_json = json.loads(r.text)

if 200 == r.status_code:

if response_json['acknowledged']:

logging.info("index_name: %s 创建成功" % (index_name))

new_index_array.append(index_name)

if do_reindex:

result = es_reindex_by_rel_type(source_index, index_name, relation_type, routing_field)

if 0 != result:

logging.info("do new_index_array:%s over" % (new_index_array))

return -1

else:

logging.info("index_name: %s 创建失败" % (index_name))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

logging.info("do new_index_array:%s over" % (new_index_array))

return -1

logging.info("index create done!")

logging.info("new_index_array:%s" % (new_index_array))

return 0

# 备份索引内容(结构+数据)

def es_reindex_by_rel_type(source_index, bak_index, rel_type, routing_field):

# 备份数据

url = es_url + "_reindex"

data = {

"source": {

"index": source_index,

"query": {

"term": {

"rel_type": rel_type.swapcase()

}

}

},

"dest": {

"index": bak_index

},

"script": {

"inline": "ctx._routing = ctx._source." + routing_field,

"lang": "painless"

}

}

r = requests.post(url, json.dumps(data), headers={"content-type": "application/json"})

if 200 == r.status_code:

response_json = json.loads(r.text)

logging.info("备份数据返回响应： %s " % r.text.encode(encoding="utf-8"))

logging.info("备份索引： %s , 备份数据量： %d" % (bak_index, response_json['total']))

else:

logging.info("source_index:%s bak_index: %s 备份失败" % (source_index, bak_index))

logging.error("错误信息： %s" % r.text.encode(encoding="utf-8"))

return -1

return 0

# 分页查询

def get_es_data_by_scroll(es_url, index, query={}, scroll_id=None, scroll="5m", batch_size=10000):

data = []

try:

while True:

if not scroll_id:

# 每次取的数据量

query["size"] = batch_size

curl_url = es_url + index + '/_search?scroll=' + scroll

logging.info("curl_url:%s" % (curl_url))

response = requests.post(curl_url, json.dumps(query), headers={'content-type': 'application/json'})

else:

curl_url = es_url + '_search/scroll?scroll=' + scroll + '&scroll_id=' + scroll_id

logging.info("curl_url:%s" % (curl_url))

response = requests.get(curl_url)

# 结果返回处理

if response:

if 200 == response.status_code:

response_json = json.loads(response.text)

scroll_id = response_json['_scroll_id']

# Update the scroll ID

if scroll_id is None:

break

# Get the number of results that we returned in the last scroll

if not response_json['hits']['hits']:

break

response_data = [doc["_source"] for doc in response_json['hits']['hits']]

data.extend(response_data)

else:

logging.info("curl_url:%s 查询失败" % (curl_url))

logging.error("错误信息： %s" % response.text.encode(encoding="utf-8"))

return None

else:

logging.info("curl_url:%s 查询失败" % (curl_url))

return None

logging.info("get data size:%s" % (len(data)))

return data

except Exception as e:

logging.error(e)

logging.error("query fail!")

print("exception!")

return None

# 查询

def query(es_url, index, query, batch_size=1000, scroll="3m"):

try:

curl_url = es_url + index_name + '/_search?scroll=' + scroll

# 每次取的数据量

query["size"] = batch_size

response = requests.post(curl_url, json.dumps(query), headers={'content-type': 'application/json'})

if response:

response_json = json.loads(response.text)

return response_json

except Exception as e:

logging.error(e)

logging.error("query fail!")

print("exception!")

# 分页查询

def query_by_scroll(es_url, index, doc_type=None, query=None, scroll='5m', batch_size=1000):

'''

:param index:

:param doc_type:

:param query:

queryParam = {

"query": {

"bool": {

"must": [

{

"range": {

"import_es_time": {

"lt": "2019-07-31 00:00:00"

}

}

},

{

"term": {

"list_type": "01"

}

},

{

"term": {

"delete_status": "0"

}

}

],

"must_not": [],

"should": []

}

}

}

:param scroll:

:param batch_size:

:return:

'''

try:

logging.info("query: index:%s doc_type:%s scroll:%s batch_size:%s query:%s" % (

index, doc_type, scroll, batch_size, query))

# 每次取的数据量

query["size"] = batch_size

if doc_type:

curl_url = es_url + index + '/' + doc_type + '/_search?scroll=' + scroll

else:

curl_url = es_url + index + '/_search?scroll=' + scroll

response = requests.post(curl_url, json.dumps(query), headers={'content-type': 'application/json'})

if response:

if 200 == response.status_code:

response_json = json.loads(response.text)

return response_json

else:

logging.info("curl_url:%s query: %s 失败" % (curl_url, query))

logging.error("错误信息： %s" % response.text.encode(encoding="utf-8"))

return None

except Exception as e:

logging.error(e)

logging.error("query fail!")

print("query_by_scroll exception!")

return None

# 分页scroll_id查询

def query_by_scroll_id(es_url, index, scroll_id, scroll='5m'):

if scroll_id is None:

return

try:

curl_url = es_url + '_search/scroll?scroll=' + scroll + '&scroll_id=' + scroll_id

logging.info("curl_url:%s" % (curl_url))

response = requests.get(curl_url)

# 结果返回处理

if response:

if 200 == response.status_code:

response_json = json.loads(response.text)

return response_json

else:

logging.info("curl_url:%s 查询失败" % (curl_url))

logging.error("错误信息： %s" % response.text.encode(encoding="utf-8"))

return None

else:

logging.info("curl_url:%s 查询失败" % (curl_url))

return None

except Exception as e:

logging.error(e)

logging.error("query fail! scroll_id:%s" % (scroll_id))

print("query_by_scroll_id exception!")

return None

# 分页获取解析数据

def get_and_parse_query_scroll_data(es_url, index, doc_type=None, query=None, scroll='5m', batch_size=1000,

convert_dict={}, add_date_time=False):

page = query_by_scroll(es_url, index, doc_type=doc_type, query=query, scroll=scroll)

return convert_es_page_data(page, convert_dict, add_date_time)

# 解析返回数据

def parse_es_page_data(page):

result_data = {}

if not page or not page['_scroll_id']:

logging.warning("query_by_scroll return none")

print("query_by_scroll return none")

return result_data

if page['_scroll_id']:

scroll_id = page['_scroll_id']

result_data[PAGE_RESULT_SCROLL_ID] = scroll_id

print("Scrolling scroll_id:%s" % (scroll_id))

if page['hits']:

total_size = page['hits']['total']

print("Scrolling total_size:%s" % (total_size))

result_data[PAGE_RESULT_TOTAL_SIZE] = total_size

hits = page['hits']['hits']

scroll_size = len(hits)

result_data[PAGE_RESULT_SCROLL_SIZE] = scroll_size

result_data[PAGE_RESULT_HITS] = hits

return result_data

# 根据业务需要转换数据

def convert_es_page_data(page, convert_dict={}, add_date_time=False):

'''

:param page:

:param convert_dict:

convert_dict 示例

{"key1": {"dest_key": ["key1","key2"], "default_value":""}}

:param add_date_time:

:return:

'''

result_data = parse_es_page_data(page)

if result_data and result_data['hits']:

result = result_data['hits']

# parse data

convert_data = []

for item in result:

if item['_source']:

source_data = item['_source']

convert_result = {}

keys = convert_dict.keys()

for source_key in keys:

dest_dict = convert_dict.get(source_key, [])

dst_keys = dest_dict.get(CONVERT_DEST_KEY, [])

default_value = dest_dict.get(CONVERT_DEFAULT_VALUE, '')

for dst_key in dst_keys:

convert_result[dst_key] = source_data.get(source_key, default_value)

if add_date_time:

date_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))

convert_result["date_time"] = date_time

convert_str = json.dumps(convert_result, ensure_ascii=False)

convert_data.append(convert_str.encode('utf-8'))

result_data[PAGE_RESULT_CONVERT_DATA] = convert_data

return result_data

def main():

# ES服务器地址

# ##开发环境

es_url = "http://192.168.3.63:9200/"

# ##测试环境

# es_url = "http://192.168.3.206:9200/"

# ##预发环境

# es_url = "http://100.1.1.1:9200/"

# ##线上环境

# es_url = "http://10.1.1.1:9200/"

BUCKET_MUM = 2

INDEX_NAME = 'zyc_test'

DOC_TYPE = 'relation'

BAK_INDEX_NAME = 'backup' + INDEX_NAME

INDEX_PREFIX = INDEX_NAME + '_'

BAK_INDEX_PREFIX = BAK_INDEX_NAME + '_'

ES_MAPPING = {

"settings": {

"index": {

"search": {

"slowlog": {

"threshold": {

"fetch": {

"warn": "100ms"

},

"query": {

"warn": "100ms"

}

}

}

},

"refresh_interval": "1s",

"indexing": {

"slowlog": {

"threshold": {

"index": {

"warn": "1s"

}

}

}

},

"number_of_shards": "6",

"translog": {

"flush_threshold_size": "1gb",

"sync_interval": "120s",

"durability": "async"

}

}

},

"aliases": {

"vii_relation": {

}

},

"mappings": {

"relation": {

"dynamic_date_formats": [

"yyyy-MM-dd HH:mm:ss",

"yyyy-MM-dd"

],

"dynamic_templates": [

{

"date_template": {

"match_pattern": "regex",

"mapping": {

"format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis",

"type": "date"

},

"match_mapping_type": "string",

"match": "(.*_date|.*_timestamp|.*_ts|.*_time)"

}

},

{

"keyword_template": {

"mapping": {

"type": "keyword"

},

"match_mapping_type": "string"

}

}

],

"_all": {

"enabled": "false"

},

"properties": {

"pk": {

"type": "keyword"

},

"in_obj_type": {

"type": "keyword"

},

"in_obj_value": {

"type": "keyword"

},

"in_obj_type_value": {

"type": "keyword"

},

"in_obj_tag": {

"type": "keyword"

},

"in_obj_tag_desc": {

"type": "keyword"

},

"out_obj_type": {

"type": "keyword"

},

"out_obj_value": {

"type": "keyword"

},

"out_obj_type_value": {

"type": "keyword"

},

"out_obj_tag": {

"type": "keyword"

},

"out_obj_tag_desc": {

"type": "keyword"

},

"start_time": {

"type": "date",

"format": "yyyy-MM-dd HH:mm:ss"

},

"end_time": {

"type": "date",

"format": "yyyy-MM-dd HH:mm:ss"

},

"rel_type": {

"type": "keyword"

},

"rel_detail": {

"type": "keyword",

"index": "false"

},

"count": {

"type": "long"

},

"similarity": {

"type": "double"

},

"tag_codes": {

"type": "keyword"

},

"delete_status": {

"type": "integer"

},

"tenant_code": {

"type": "keyword"

},

"business_code": {

"type": "keyword"

},

"import_es_time": {

"type": "date",

"format": "yyyy-MM-dd HH:mm:ss"

}

}

}

}

}

ADD_ES_MAPPING = {

"properties": {

"delete_status": {

"type": "integer"

},

"start_time": {

"format": "yyyy-MM-dd HH:mm:ss",

"type": "date"

},

"end_time": {

"format": "yyyy-MM-dd HH:mm:ss",

"type": "date"

}

}

}

queryParam = {

"query": {

"bool": {

"must": [

],

"must_not": [],

"should": []

}

}

}

QUERY_INDEX_NAME = 'cysaas_object_basic_info'

QUERY_DOC_TYPE = 'basic'

logging.info("begin...")

create_index(es_url, INDEX_NAME, ES_MAPPING)

time.sleep(5)

delete_index(es_url, INDEX_NAME)

time.sleep(5)

batch_create_index_with_mapping(es_url, INDEX_PREFIX, ES_MAPPING)

time.sleep(5)

add_properties_to_index(es_url, INDEX_NAME, DOC_TYPE, ADD_ES_MAPPING)

time.sleep(5)

batch_add_properties_to_index(es_url, INDEX_PREFIX, DOC_TYPE, ADD_ES_MAPPING, 2)

time.sleep(5)

result = query_by_scroll(es_url, index=QUERY_INDEX_NAME, query=queryParam)

convert_dict = {"obj_value": {"dest_key": ["obj_value", "value"], "default_value": ""}}

result_data = get_and_parse_query_scroll_data(es_url, index=QUERY_INDEX_NAME, query=queryParam,

convert_dict=convert_dict)

get_data = get_es_data_by_scroll(es_url, index=QUERY_INDEX_NAME, query=queryParam)

del_index_by_prefix(es_url, INDEX_NAME)

time.sleep(5)

batch_es_del_index(es_url, INDEX_PREFIX, 2)

logging.info("done")

print("done")

if __name__ == '__main__':

main()

二、ES Python脚本查询操作 client方式

#!coding:utf-8

import json

import logging

import time

from elasticsearch import Elasticsearch, helpers

PAGE_RESULT_SCROLL_ID = 'scroll_id'

PAGE_RESULT_SCROLL_SIZE = 'scroll_size'

PAGE_RESULT_TOTAL_SIZE = 'total_size'

PAGE_RESULT_HITS = 'hits'

PAGE_RESULT_DATA = 'data'

PAGE_RESULT_CONVERT_DATA = 'convert_data'

CONVERT_DEST_KEY = 'dest_key'

CONVERT_DEFAULT_VALUE = 'default_value'

current_time = time.strftime("%Y-%m-%d-%H-%M", time.localtime(time.time()))

# 日志设置

log_file = "operate_es_client_" + current_time + ".log"

logging.FileHandler(filename=log_file, encoding='utf-8')

logging.basicConfig(filename=log_file, level=logging.INFO)

def query(es_client, index, query):

try:

return helpers.scan(es_client, index=index, scroll="3m", query=query)

except Exception as e:

logging.error(e)

logging.error("query fail!")

print("exception!")

def query_by_scroll(es_client, index, doc_type=None, query=None, scroll='5m', batch_size=1000, preserve_order=False,

**kwargs):

'''

:param index:

:param doc_type:

:param query:

queryParam = {

"query": {

"bool": {

"must": [

{

"range": {

"import_es_time": {

"lt": "2019-07-31 00:00:00"

}

}

},

{

"term": {

"list_type": "01"

}

},

{

"term": {

"delete_status": "0"

}

}

],

"must_not": [],

"should": []

}

}

}

:param scroll:

:param batch_size:

:param preserve_order:

:param kwargs:

:return:

'''

if not preserve_order: # 是否需要scan模式

kwargs['search_type'] = 'query_then_fetch'

try:

logging.info("query: index:%s doc_type:%s scroll:%s batch_size:%s query:%s" % (

index, doc_type, scroll, batch_size, query))

resp = es_client.search(index=index,

doc_type=doc_type,

scroll=scroll,

size=batch_size,

body=query,

**kwargs)

return resp

except Exception as e:

logging.error(e)

logging.error("query fail!")

print("exception!")

return None

def query_by_scroll_id(es_client, scroll_id, scroll='5m'):

if scroll_id is None:

return

try:

resp = es_client.scroll(scroll_id, scroll=scroll)

return resp

except Exception as e:

logging.error(e)

logging.error("query fail! scroll_id:%s" % (scroll_id))

print("exception!")

return None

def get_and_parse_query_scroll_data(es_client, index, doc_type=None, query=None, scroll='5m', batch_size=1000,

convert_dict={}, add_date_time=False):

page = query_by_scroll(es_client, index, doc_type=doc_type, query=query, scroll=scroll)

return convert_es_page_data(page, convert_dict, add_date_time)

def parse_es_page_data(page):

result_data = {}

if not page or not page['_scroll_id']:

logging.warning("query_by_scroll return none")

print("query_by_scroll return none")

return result_data

if page['_scroll_id']:

scroll_id = page['_scroll_id']

result_data[PAGE_RESULT_SCROLL_ID] = scroll_id

print("Scrolling scroll_id:%s" % (scroll_id))

if page['hits']:

total_size = page['hits']['total']

print("Scrolling total_size:%s" % (total_size))

result_data[PAGE_RESULT_TOTAL_SIZE] = total_size

hits = page['hits']['hits']

scroll_size = len(hits)

result_data[PAGE_RESULT_SCROLL_SIZE] = scroll_size

result_data[PAGE_RESULT_HITS] = hits

return result_data

def convert_es_page_data(page, convert_dict={}, add_date_time=False):

'''

:param page:

:param convert_dict:

convert_dict 示例

{"key1": {"dest_key": ["key1","key2"], "default_value":""}}

:param add_date_time:

:return:

'''

result_data = parse_es_page_data(page)

result = []

if result_data and result_data['hits']:

result = result_data['hits']

# parse data

convert_data = []

for item in result:

if item['_source']:

source_data = item['_source']

convert_result = {}

keys = convert_dict.keys()

for source_key in keys:

dest_dict = convert_dict.get(source_key, [])

dst_keys = dest_dict.get(CONVERT_DEST_KEY, [])

default_value = dest_dict.get(CONVERT_DEFAULT_VALUE, '')

for dst_key in dst_keys:

convert_result[dst_key] = source_data.get(source_key, default_value)

if add_date_time:

date_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))

convert_result["date_time"] = date_time

convert_str = json.dumps(convert_result, ensure_ascii=False)

convert_data.append(convert_str.encode('utf-8'))

result_data[PAGE_RESULT_CONVERT_DATA] = convert_data

return result_data

def main():

# ES服务器地址

# ##开发环境

es_host = "192.168.3.63"

es_url = "http://192.168.3.63:9200/"

INDEX_NAME = 'cysaas_object_basic_info'

DOC_TYPE = 'basic'

queryParam = {

"query": {

"bool": {

"must": [],

"must_not": [],

"should": []

}

}

}

logging.info("begin...")

# es_client = Elasticsearch([{'host': es_host, 'port': '9200'}])

es_client = Elasticsearch([es_url], verify_certs=False)

result = query_by_scroll(es_client, index=INDEX_NAME, doc_type=DOC_TYPE, query=queryParam)

time.sleep(5)

result_data = get_and_parse_query_scroll_data(es_client, index=INDEX_NAME, doc_type=DOC_TYPE,

query=queryParam)

logging.info("done")

print("done")

if __name__ == '__main__':

main()

scan能避免scroll的排序性能消耗，from size分页查询模式会对数据集进行整体排序, 性能损耗是很大的. 如果我们关闭排序，那么可以消耗极少资源返回所有的文档。scan就是不去，而是仅仅从每个有结果的分片中返回数据.

下面是python elasticsearch helpers.scan的源码。对照elasticsearch scroll scan基本用法，很容易就能理解下面的代码。elasticsearch-py把高性能的功能都继承在了helpers模块里，比如helpers.scan helpers.reindex streaming_bulk helpers.bulk parallel_bulk .

elasticsearch.helpers.scan(client, query=None, scroll=u'5m', raise_on_error=True, preserve_order=False, **kwargs)

参数介绍:

client – elasticsearch的连接对象

query – elasticsearch dsl查询语句

scroll – 你想让scroll的结果集在server端标记多久

raise_on_error – raise的error class

preserve_order – 这里其实对应的是search_type，是否要求排序

file: helpers/__init__.py

#默认是5m分钟， 默认是search_type是scan扫描模式

def scan(client, query=None, scroll='5m', preserve_order=False, **kwargs):

if not preserve_order: #是否需要scan模式

kwargs['search_type'] = 'scan'

resp = client.search(body=query, scroll=scroll, **kwargs)

scroll_id = resp.get('_scroll_id') #第一次查询拿到_scroll_id token

if scroll_id is None:

return

first_run = True

while True:

#如果你server_type不是scan，那么第一次的结果里是包含数据的。

if preserve_order and first_run:

first_run = False

else:

resp = client.scroll(scroll_id, scroll=scroll)

if not resp['hits']['hits']:

break

for hit in resp['hits']['hits']:

yield hit #通过yield生成器来返回数据

scroll_id = resp.get('_scroll_id')

if scroll_id is None:

break

file: client/__init__.py

@query_params('scroll')

def scroll(self, scroll_id, params=None):

# 第二次scroll的数据请求是直接 /_search/scroll,方法用的是GET

_, data = self.transport.perform_request('GET', '/_search/scroll',params=params, body=scroll_id)

return data

对于elasticsearch scanscroll的使用方法, 大家注意一下异常情况.

data = scan(es,

query={"query": {"match": {"domain": "xiaorui.cc"}}},

index="xiaorui_index",

doc_type="blog"

)

for one in data:

print one

Elasticsearch博大精深… … 经过我的线下线上测试，使用scroll scan的性能还是不错的，返回的速度不错，明显比那种from size分页要快速，而且节省了elasticsearch的检索资源。

weixin_39572409
关注
  • 0
    点赞
  • 踩
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
Python-elasticsearchPython脚本
08-11
elasticsearch Python脚本
python elasticsearch 查询简单脚本
qq_41470573的博客
06-17 340
入职的新公司，中台使用的elasticsearch给我们提供的数据，已经测了两三个需求了，但不清楚如何查询数据，用例评审时直接说的不校验数据准确性，只做展示校验。然后，上线后发现了一堆bug，虽说没我啥z ...
Elasticsearch python 启动脚本
ywmack
11-14 520
import os import sys def endpcess(): nb= os.popen("jps|grep Elasticsearch").read() oldid=nb.split(' ')[0] #print 'oldid'+oldid nb=os.system("kill "+oldid) return 'Elasticsearch i...
python编写es脚本_Python 批量写入 Elasticsearch 脚本
weixin_39683368的博客
11-24 152
Elasticsearch官方和社区提供了各种各样的客户端库，在之前的博客中，我陆陆续续提到和演示过Perl的，Javascript的，Ruby的。上周写了一版Python的，考虑到好像很难找到现成的示例，如何用python批量写数据进Elasticsearch，今天一并贴上来。#!/usr/bin/env pypy#coding:utf-8import reimport sysimport ti...
python编写es脚本_python中的Elasticsearch操作汇总
weixin_39561673的博客
11-24 203
这篇文章主要介绍了python中的Elasticsearch操作汇总,文中通过示例代码介绍的非常详细，对大家的学习或者工作具有一定的参考学习价值,需要的朋友可以参考下导入包from elasticsearch import Elasticsearch本地连接es = Elasticsearch(['127.0.0.1:9200'])创建索引es.indices.create(index="pyth...
Python 脚本获取ES 存储容量的实例
12-23
最近有需求统计ES存储容量，之前用PHP实现的，考虑到以后可能会经常写脚本查询，故用python写了一个脚本，代码如下： import urllib import urllib2 import sys es_service_addr = sys.argv[1] url = ...
基于python的elasticsearch和gurobi的外卖配送路径优化问题
05-19
通过编写Python脚本, 实现了利用elasticsearch读取大型数据, 然后通过gurobi对模型进行求解, 最后将最优路径通过Folium生成HTML文件展示. 该资源包括模型说明（英文，bd翻译即可）、完整的数据、源代码、运行结果...
mongodb-elasticsearch-connector:将 MongoDB 集合导出到 Elasticsearch 的 Python 脚本
05-29
mongodb-elasticsearch-connector 将 MongoDB 集合导出到 Elasticsearch 的 Python 脚本。用法所有配置选项都可以设置为命令行参数或环境变量。 至少必须设置collection和database 。 可以使用mongodb_elasticsearch...
python Elasticsearch索引建立和数据的上传详解
12-31
今天我想讲一讲关于Elasticsearch的索引建立，当然提前是你已经安装部署好Elasticsearch。 ok，先来介绍一下Elaticsearch，它是一款基于lucene的实时分布式搜索和分析引擎，是后台系统，用来存储数据，检索数据，...
Python用于清理elasticsearch脚本~
经纬的博客
01-07 346
王婆卖瓜，自卖自夸一下，这代码越写越规范了，真香！ 用于清理elasticsearch数据，放着自己记录看一下 #!/usr/bin/env python3 # -*- coding: utf-8 -*- ''' elasticsearch clean ''' __author__ = 'jiangjw' import requests import heapq import traceback...
实习工作复盘·用Elasticsearch的Python的API实现搜索功能
My Struggle
03-22 2788
之前几日，我先实现了用Python脚本导入XML文件到Elasticsearch中。现在我又写了一个初步的脚本，用elasticsearch框架实现了检索的功能，下面的想法可能是做成一个WEB端的检索系统，目前先实现相关的后端功能。 具体要求和思路 上面给的要求是输入有两种列表，除此之外不会有第三种序列： [str1,str2,······,strN] 这种列表的语义是str1OR str2 OR······strN [(str1,str2),(str3,str4),······(strN-1,..
ES _reindex实践初步总结
BD_fuhong的博客
12-30 622
1、建立新索引 2、指定reindex { "source": { "index": "deviceinfo_es" }, "dest": { "index": "deviceinfo" } } 结果如图： 3.2G,77万条数据，耗时526324毫秒，约10分钟。 参考： https://blog.csdn.net/laoyang36...
python es查询
王小二的博客
10-24 3440
python es查询
Elasticsearch模块功能之-脚本语言（Scripting）
路漫漫其修远兮
08-05 7311
使用脚本语言可以计算自定义表达式的值，比如计算自定义查询相关度评分。支持的脚本语言有groovy，js，mvel（1.3.0废弃），python等。  脚本文件必须放在在配置文件conf目录下，如config/scripts/calculate.python，如果文件路径是config/scripts/group1/group2/test.py则引用名称为group1_group2_test。
Python编写Oracle和Elasticsearch数据同步脚本
JasonYSU的博客
07-19 4143
python版本 2.7.12 64位 Oracle（x64 12.10.2）和Elasticsearch（2.2.0）位于远程服务器
python Elasticsearch 遍历相关数据
wangqing008的专栏
11-28 3827
一开始用的是这个 res = es.search( index='xxxxx*/', #_source=['localtime', 'request'], body=actor_count_json, search_type="query_then_fetch", scroll="1m" ) sc
ElasticSearch 开发总结（九）——SearchType：DFS_QUERY_THEN_FETCH和QUERY_THEN_FETCH
腊八粥
12-10 5950
1.SearchType ES的搜索类型 有一个类SearchType（如下图示），关于该类的描述： Search type represent the manner at which the search operation is executed. ES的搜索分scatter/gather两个步骤： scatter阶段:客户端向10个分片发起搜索请求； gather阶段:10个分片完...
preserve-order=true 无效的原因分析
梦兰
03-14 1354
       使用testNG suite测试的时候，如果一个<test>包含多个class，你又想指定执行的顺序，通常的做法是加入这样的属性<test name="testng_recharge3" preserve-order="true"> ，但是实际使用中要注意，如果你的class中包含有BeforeTest方法，就算你指定了preserve-order="true...
使用python库elasticsearch操作es
buside的博客
11-19 1778
1、单个写入 from elasticsearch import Elasticsearch es = Elasticsearch("192.168.0.11:9200") mappings = { "id": "11", "serial": "版本", "tags": {"comment": "标签3"}, "status": "tag1" } es.index(index="index-te
编写python脚本实现es集群扩容
最新发布
05-25
实现ES集群扩容的python脚本可以使用Elasticsearch官方Python客户端库进行编写。以下是一个简单的示例脚本： ```python from elasticsearch import Elasticsearch # 创建Elasticsearch客户端实例 es = Elastic...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值