通过ES查询指定条件的全量数据python实现

from elasticsearch import Elasticsearch

def test():
    try:
        l_time = datetime.datetime.now() + datetime.timedelta(minutes=-15)
        now_time = util.local2utc(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'))
        now_time_5m = util.local2utc(l_time.strftime('%Y-%m-%d %H:%M:%S.%f'))
        es = Elasticsearch([{"host": "192.168.25.30", "port": "9200"}])
        body = {
            "query": {
                "bool": {
                    "must": {
                        "match_all": {}
                    },
                    "filter": {
                        "range": {
                            "@timestamp": {
                                "gte": now_time_5m,
                                "lte": now_time
                            }
                        }
                    }
                }
            },
            "size": 10000,
            "sort": {
                "@timestamp": {"order": "asc"}
            },
            "_source": ["status", "method", "client_ip", "remote_ip", "timestamp", "request_time", "@timestamp"]
        }
        queryData = es.search(index='your_index_name', scroll='5m', timeout='3s', size=10000, body=body)
        mdata = queryData.get("hits").get("hits")
        if not mdata:
            print 'empty!'
        scroll_id = queryData["_scroll_id"]
        total = queryData["hits"]["total"]
        print "total is :%s"%total
        for i in range(total / 10000):
            res = es.scroll(scroll_id=scroll_id, scroll='5m')  # scroll参数必须指定否则会报错
            mdata += res["hits"]["hits"]
        print len(mdata)
    except Exception as ex:
        print "test function excute exception:" + str(ex)


if __name__ == "__main__":
    test()

转载于:https://www.cnblogs.com/chmyee/p/9946254.html