python之curl命令读取elasticsearch数据

在shell中curl命令格式要求严格，细节之处以记录之

from subprocess import Popen, PIPE
import json
import codecs
import os
count = 0
file_count = 0
for i in range(600):
    cmd_shell= u"curl -XGET -H \"Content-Type:application/json\" -u elastic:aJh4HAugLsHT  es-cn-mp90zgtkc000kh34w.public.elasticsearch.aliyuncs.com:9200/content_pool_v3/_search?scroll=10m -d '{\"from\":\"%s\",\"size\":\"1000\"}\'"%(count)
    p = Popen(cmd_shell, stdout=PIPE,stderr=PIPE,stdin=PIPE,shell=True)
    json_data = json.loads(p.stdout.read())
    _scroll_id = json_data['_scroll_id']
    json_data_hits = json_data['hits']['hits']
    for i in range(len(json_data_hits)):
        title = json_data_hits[i]['_source']['title']
        summary = json_data_hits[i]['_source']['summary']
        region_tags = ",".join(json_data_hits[i]['_source']['region_tags'])
        try:
            filename = str(file_count) + "_summary_.txt"
            with codecs.open(filename, "a+", 'utf-8') as wfp:
                if round(os.path.getsize(filename) / float(1024 * 1024), 2) > 10:
                    file_count = file_count + 1
                wfp.write(title + u"。" + u"\n")
                wfp.write(summary + u"\n")
                wfp.write(region_tags + u"。" + u"\n")
        except:
            pass


    while True:
        cmd_shell = u"curl -XGET -H \"Content-Type:application/json\" -u elastic:aJh4HAugLsHT es-cn-mp90zgtkc000kh34w.public.elasticsearch.aliyuncs.com:9200/_search/scroll -d '{\"scroll\":\"10m\",\"scroll_id\":\"" + str(_scroll_id) + u"\"}'"
        p = Popen(cmd_shell,stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=True)
        json_data = json.loads(p.stdout.read())
        if '_scroll_id' in json_data:
            _scroll_id = json_data['_scroll_id']
        else:
            break
        json_data_hits = json_data['hits']['hits']
        for i in range(len(json_data_hits)):
            title = json_data_hits[i]['_source']['title']
            print(title)
            summary = json_data_hits[i]['_source']['summary']
            region_tags = ",".join(json_data_hits[i]['_source']['region_tags'])
            try:
                filename = str(file_count) + "_summary_.txt"
                with codecs.open(filename, "a+", 'utf-8') as wfp:
                    if round(os.path.getsize(filename) / float(1024 * 1024), 2) > 10:
                        file_count = file_count + 1
                    wfp.write(title + u"。" + u"\n")
                    wfp.write(summary + u"\n")
                    wfp.write(region_tags + u"。" + u"\n")
            except:
                pass
        #content = i_json_data['_source']['content']
    count = count + 1000