使用commons-httpclinet 请求ES数据

使用commons-httpclinet 请求ES数据

        使用Java请求es，一般使用RestHighLevelClient 进行请求查询，其通用性会很好。这边使用Commons-httpclient里面的 HttpClient进行请求，这时候拼接参数查询就直接是es的语法了。

直接上代码：

代码：

1，引用：

引用文件：

gradle:

implementation 'commons-httpclient:commons-httpclient:3.1'

Maven:

<dependency>
    <groupId>commons-httpclient</groupId>
    <artifactId>commons-httpclient</artifactId>
    <version>3.1</version>
</dependency>

Yml:

es:
    elasticSearchUrl: http://ip:port
	xpack:
		security:
			user: elastics:123456 
			enabled: false

2，CommonsEsQueryHttpClient

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cloud.context.config.annotation.RefreshScope;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import java.io.IOException;
import java.io.InputStream;


@Service
@RefreshScope
@Slf4j
public class CommonsEsQueryHttpClient {

    @Value("${es.elasticSearchUrl}")
    private String url;

    @Value("${es.xpack.security.enable}")
    private String xpack;

    @Value("${es.xpack.security.user}")
    private String user;

    private static HttpClient httpClient;
    private static final int MAX_HOST_CONNECTIONS = 10;
    private static final int MAX_TOTAL_CONNECTIONS = 20;


    @PostConstruct
    public void init() {
        MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
        HttpConnectionManagerParams params = new HttpConnectionManagerParams();
        params.setDefaultMaxConnectionsPerHost(MAX_HOST_CONNECTIONS);
        params.setMaxTotalConnections(MAX_TOTAL_CONNECTIONS);
        connectionManager.setParams(params);
        httpClient = new HttpClient(connectionManager);
        httpClient.getParams().setParameter(HttpMethodParams.HTTP_CONTENT_CHARSET, "UTF-8");
        if ("true".equals(xpack)) {
            String esXPackUserName = StringUtils.substringBefore(user, ":");
            String esXPackPasswd =  StringUtils.substringAfter(user, ":");
            httpClient.getState().setCredentials(AuthScope.ANY,
                    new UsernamePasswordCredentials(esXPackUserName, esXPackPasswd));
        }
    }

    private String postMethod(String index, String requestParamData) throws IOException {
        PostMethod postMethod = new PostMethod(this.url + index);
        InputStream in = null;
        try {
            RequestEntity requestEntity = new StringRequestEntity(requestParamData, null, null);
            postMethod.setRequestEntity(requestEntity);
            postMethod.setRequestHeader("Content-Type", MediaType.APPLICATION_JSON_UTF8_VALUE);
            int statusCode = httpClient.executeMethod(postMethod);
            if (statusCode == HttpStatus.SC_OK) {
                in = postMethod.getResponseBodyAsStream();
                return IOUtils.toString(in);
            }
        } finally {
            postMethod.releaseConnection();
            if (in != null) {
                in.close();
            }
        }
        return null;
    }

    /**
     * 获取es中数据
	 * @param index 查询的索引 
	 * @param requestParamData 查询参数
     */
public String getEsData(String index,String requestParamData) throws IOException
{
        return postMethod(index, requestParamData);
    }
}

 

3，值处理类：

mport com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.collections4.ListUtils;
import org.apache.commons.collections4.MapUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;

@Component
public class EsQueryUtils {

    private static Logger logger = LoggerFactory.getLogger(EsQueryUtils.class);

    @Resource
    private CommEsQueryClient commEsQueryClient;

    public static final Integer HITS_MAX_SIZE = 9999;
    public static final String ES_SCROLL_10M = "10m";
    public static final String ES_SCROLL_5M = "5m";


    public static final String KEY = "key";
    public static final String COUNT = "doc_count";
    public static final String BUCKETS = "buckets";
    public static final String TAGS = "all_tags";
    public static final String HITS = "hits";


    public JSONObject getEsDataForArgs(String index, JSONObject queryParam) {
        logger.info("ES查询索引 =====" + index);
        logger.info("ES查询参数 =====" + JSON.toJSONString(queryParam));
        commEsQueryClient.init();
        String esData;
        try {
            esData = commEsQueryClient.getEsData(index, queryParam.toJSONString());
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return JSON.parseObject(esData);
    }

    public JSONObject getEsData(String indexName, JSONObject queryParam) {
        return getEsDataForArgs("/" + indexName + "/_search", queryParam);
    }



    public JSONArray getEsBuckets(String tags, JSONObject esData) {
        return Optional.ofNullable(esData).map(e -> e.getJSONObject("aggregations")).map(agg -> agg.getJSONObject(tags))
                .map(tag -> tag.getJSONArray(BUCKETS)).orElse(new JSONArray());
    }

    public Long getEsTotalCount(JSONObject esData) {
        return Optional.ofNullable(esData).map(e -> e.getJSONObject(HITS)).map(hits -> hits.getLongValue("total"))
                .orElse(0L);
    }

    public JSONArray getEsHis(JSONObject esData) {
        return Optional.ofNullable(esData).map(e -> e.getJSONObject(HITS)).map(his -> his.getJSONArray(HITS))
                .orElse(new JSONArray());
    }

    public JSONArray getEsHisSource(JSONObject esData) {
        JSONArray jsonArray = Optional.ofNullable(esData).map(e -> e.getJSONObject(HITS))
                .map(his -> his.getJSONArray(HITS)).orElse(new JSONArray());
        JSONObject temp;
        JSONArray sourceArray = new JSONArray();
        for (int i = 0; i < jsonArray.size(); i++) {
            temp = jsonArray.getJSONObject(i).getJSONObject("_source");
            sourceArray.add(temp);
        }
        return sourceArray;
    }

    public Map<String, JSONObject> formatEsBuckets(JSONArray buckets, String key) {
        return ListUtils.emptyIfNull(buckets).stream().map(e -> (JSONObject) e).collect(HashMap::new,
                (m, v) -> m.put(MapUtils.getString(v, key), v), HashMap::putAll);
    }

 
}

总结：

    使用commons-httpclinet的HttpClient查询es数据，处理查询参数和处理值会方便些。目前我还不懂HttpClient跟RestHighLevelClient 的实质的区别。后面了解了，再添加解释说明。

《使用RestHighLevelClient 请求ES数据》