说明:当数据量比较大的时候,可采用此方法来全量获取数据
代码示例:
# -*- coding: utf-8 -*-
"""
Created on Thu Jun 7 10:08:02 2018
@author: EDZ
"""
from elasticsearch import Elasticsearch
from elasticsearch import helpers
def GetVlue():
es = Elasticsearch(["192.168.0.89:9200"])
query ={
"query": {"match_all": {}}
}
scanResp = helpers.scan(client=es, query=query, scroll="3m", index='ect_china_pv_news', doc_type='pv_news', timeout="3m")
for k in scanResp:
yield k
def write_file(k):
with open('China_news.csv','ab') as f:
k=dict(k)
f.write(k['_source']['time'].encode('gbk'))
f.write(b',')
f.write(k['_source']['title'].encode('gbk'))
f.write(b',')
f.write(str(k['_source']['positive_index']).encode('gbk'))
f.write(b',')
f.write(str(k['_source']['confidence_index']).encode('gbk'))
f.write(b',')
f.write(str(k['_source']['negative_index']).encode('gbk'))
f.write(b',')
f.write(k['_source']['headline'].encode('gbk'))
f.write(b',')
f.write(k['_source']['Keyword1'].encode('gbk'))
f.write(b',')
f.write(str(k['_source']['Keyword1_proportion']).encode('gbk'))
f.write(b',')
f.write(k['_source']['Keyword2'].encode('gbk'))
f.write(b',')
f.write(str(k['_source']['Keyword2_proportion']).encode('gbk'))
f.write(b'\n')
if __name__=="__main__":
list1 = GetVlue()
for index,k in enumerate(list1,1):
write_file(k)
print('正在导出'+str(index)+'条数据')