import json import time import requests class exportEsData(): size = 10000 def __init__(self, url,index,type): self.url = url+"/"+index+"/"+type+"/_search" self.index = index self.type = type def exportData(self): print("export data begin...") begin = time.time() msg = requests.get(self.url).text print(msg) obj = json.loads(msg) num = obj["hits"]["total"] start = 0 end = num/self.size+1 while(start<end): msg = requests.get(self.url+"?from="+str(start*self.size)+"&size="+str(self.size)).text self.writeFile(msg) start=start+1 print("export data end!!!\n\t total consuming time:"+str(time.time()-begin)+"s") def writeFile(self,msg): obj = json.loads(msg) print(obj) vals = obj["hits"]["hits"] try: f = open(self.index+"_"+self.type+".json","a") for val in vals: a = json.dumps(val["_source"],ensure_ascii=False) f.write(a+"\n") finally: f.flush() f.close() if __name__ == '__main__': exportEsData(host,index,type).exportData() 需要注意的是当es数据量大于1w时会报错,解决方法详见上一篇博文
从es中批量导出数据
最新推荐文章于 2024-05-10 11:51:26 发布