try:
es_scroll_size = 1000
with Elasticsearch(
[self.hosts], # 连接集群,以列表的形式存放各节点的IP地址
sniff_on_start=True, # 连接前测试
sniff_on_connection_fail=True, # 节点无响应时刷新节点
sniff_timeout=60 # 设置超时时间
) as es_client:
search_data = es_client.search(index=self.indexname, size=es_scroll_size, scroll='5m', body={
"query": {
"bool": {
"must": [
{"term": {
"name": {
"value": self.name
}
}},
{"range": {
"datetime": {
"gte": self.dt_start,
"lte": self.dt_to
}
}}
]
}
},
"sort": [
{
"datetime": {
"order": "asc"
}
}
]
})
hits_source = search_data['hits']['hits']
# es查询出的结果总量
total = search_data['hits']['total']
# 游标用于输出es查询出的所有结果
scroll_id = search_data['_scroll_id']
for i in range(0, int(total['value'] / es_scroll_size) + 1):
# scroll参数必须指定否则会报错
query_scroll = es_client.scroll(scroll_id=scroll_id, scroll='5m')['hits']['hits']
hits_source += query_scroll
if hits_source and len(hits_source) > 0:
source_arr = []
for source in hits_source:
source_arr.append(source['_source'])
self.data = source_arr
except Exception as e:
print(e)
python连接elasticsearch,并通过scroll查询满足条件的全部数据
最新推荐文章于 2024-06-27 09:22:21 发布