<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>5.5.3</version>
</dependency>
<!--对应使用的es版本,下面的包不需要对应-->
<dependency>
<groupId>io.searchbox</groupId>
<artifactId>jest</artifactId>
<version>6.3.1</version>
</dependency>
elasticsearch:
jest:
uris: http://*****************:19210
connection-timeout: 9000
read-timeout: 9000
multi-threaded: true
一.大量数据方式一
@Autowired
JestClient jestClient;
//实现
List<EsEntity> entityListAll = new ArrayList<>();
try {
/* 工厂创建链接 ,可不用,应为引入了jar包
JestClient jestClient1=jestClient;
try {
if(jestClient==null){
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new HttpClientConfig
.Builder("http://localhost:9200")
.multiThreaded(true)
.build());
jestClient1=factory.getObject();
}*/
//1.构建搜索对象
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//bool
BoolQueryBuilder search = new BoolQueryBuilder();
//条件查询 MatchQueryBuilder 模糊查 TermsQueryBuilder 精准查
search.must(new MatchQueryBuilder("@type", "RT.CTS.STATION.DI"));
search.must(new MatchQueryBuilder("DATA_TYPE", "A.0001.0042.R001"));
search.must(new TermsQueryBuilder("DATA_TIME", hour));
String[] clo = {"FILE_NAME_O", "FILE_NAME_N", "type", "PROCESS_START_TIME", "IIiii", "PROCESS_STATE",
"PROCESS_END_TIME", "DATA_TYPE", "DATA_TIME", "TRAN_TIME","BUSINESS_STATE"};
//fetchSource 筛选字段 ;from(0).size(5000) 相当于MySQL的limit
searchSourceBuilder.query(search).from(0).size(5000).fetchSource(clo, null);
Search search1 = new Search.Builder(searchSourceBuilder.toString()).addIndex(tableIndex + day)
.build();
JestResult jestResult = jestClient.execute(search1);
//查询判断
if (jestResult.isSucceeded()) {
//如果查询es 成功 转换成对象
entityListAll = jestResult.getSourceAsObjectList(EsEntity.class);
}
}catch (Exception e){
System.out.println(e.getMessage());
return entityListAll;
}
return entityListAll;
方式二:引入 SearchScroll,注意配置中要把集群配置全,要不然查询结果会不全,或者把游标时间设置长一些
public List<EsEntity> getEsEntityList(String hour,String day){
List<EsEntity> entityListAll = new ArrayList<>();
try {
/*JestClient jestClient1=jestClient;
try {
if(jestClient==null){
JestClientFactory factory = new JestClientFactory();
factory.setHttpClientConfig(new HttpClientConfig
.Builder("http://localhost:9200")
.multiThreaded(true)
.build());
jestClient1=factory.getObject();
}*/
//1.构建搜索对象
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
//bool
BoolQueryBuilder search = new BoolQueryBuilder();
//条件查询
search.must(new MatchQueryBuilder("@type", "RT.CTS.STATION.DI"));
search.must(new MatchQueryBuilder("DATA_TYPE", "A.0001.0042.R001"));
search.must(new MatchQueryBuilder("DATA_TIME", hour));
String[] clo = {"FILE_NAME_O", "FILE_NAME_N", "type", "PROCESS_START_TIME", "IIiii", "PROCESS_STATE",
"PROCESS_END_TIME", "DATA_TYPE", "DATA_TIME", "TRAN_TIME","BUSINESS_STATE"};
searchSourceBuilder.query(search).fetchSource(clo, null);
Search search1 = new Search.Builder(searchSourceBuilder.toString()).addIndex(tableIndex + day).setParameter(Parameters.SIZE, 200).setParameter(Parameters.SCROLL, "1m")
.build();
JestResult jestResult = jestClient.execute(search1);
String scrollId = jestResult.getJsonObject().get("_scroll_id").getAsString();
//查询判断
if (jestResult.isSucceeded()) {
//如果查询es 成功
for (int i = 0; i < 30; i++) {
SearchScroll scroll = new SearchScroll.Builder(scrollId, "1m").build();
jestResult = jestClient.execute(scroll);
List<EsEntity> entityList = jestResult.getSourceAsObjectList(EsEntity.class);
if (!entityList.isEmpty()) {
entityListAll.addAll(entityList);
} else {
break;
}
}
}
}catch (Exception e){
System.out.println(e.getMessage());
/*try {
jestClient1.close();
} catch (IOException e1) {
System.out.println(e.getMessage());
return entityListAll;
}*/
return entityListAll;
}
return entityListAll;
}
方式三:SQL
Search search1=new Search.Builder(
"{ \n" +
" \"query\":{\n" +
" \"bool\":{\n" +
" \"must\":[\n" +
"\n" +
" {\"term\":{\n" +
" \"@type\": \"RT.CTS.STATION.DI\"\n" +
" }},\n" +
" {\"term\":{ \n" +
" \"DATA_TYPE\": \"A.0001.0042.R001\"\n" +
" }},\n" +
" {\"term\":{ \n" +
" \"DATA_TIME\": \"2021-04-24 08:00\"\n" +
" }},\n" +
" {\"terms\":{ \n" +
" \"IIiii\": [\"K1001\",\"K1002\",\"K1003\"]\n" +
" }}\n" +
" ]\n" +
" }\n" +
" },\n" +
" \"from\": 0,\n" +
" \"size\": 10\n" +
"}"
).addIndex("atlantic_log_e10adc3949ba59abbe56e057f20f88dd_20210424")
.build();
SearchResult jestResult= jestClient.execute(search1);
jestResult.isSucceeded();
//jestResult.getHits()
System.out.println(jestResult.getJsonString());