elasticsearch查询filebeat采集的日志

依赖

不要问为什么不用7或者8，因为不会

 <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>6.8.5</version>
</dependency>

ES配置

package cn.logsquery.config;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;

/**
 * @Author: junfeng
 * @CreateTime: 2022/03/18
 * @Description: es配置
 */
@ConfigurationProperties(prefix = "elasticsearch")
@Component
@Configuration
@Data
@Slf4j
public class EsConfig {

    private String ip;

    private String port;

    private String account;//账号 例：elastic

    private String passWord;//密码 例：123456

    private String fileBeatIndex;//密码 例：123456

    @Bean
    public RestHighLevelClient client() {
        log.info("~~~~~~~~~~~~~~~~~~~~~~~~~~初始化化连接ES~~~~~~~~~~~~~~~~~~~~~");
        log.info("ES信息,IP:{},PORT{},USERNAME:{},PASSWORD:{},FILEBEATINDEX:{}",ip,port,account,passWord,fileBeatIndex);
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        //设置账号密码
        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(account,passWord));
        ///创建rest client对象
        RestClientBuilder builder = RestClient.builder(new HttpHost(ip, Integer.parseInt(port))).setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
            @Override
            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
                return httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
            }
        });
        RestHighLevelClient client = new RestHighLevelClient(builder);
        return client;
    }


}

查询核心代码

 @Autowired
 private EsConfig client;

 @Override
 public IPage<LogResponseVO> queryLog(LogQueryVO logQueryVO) throws IOException {
        //1. 构建查询请求对象，指定查询的索引名称
        SearchRequest searchRequest = new SearchRequest(client.getFileBeatIndex());
        //2. 创建查询条件构建器SearchSourceBuilder
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        //3. 查询条件
        QueryBuilder queryBuilder = getBoolQuery(logQueryVO);
        //4. 指定查询条件
        sourceBuilder.query(queryBuilder);
        //5. 添加分页信息
        sourceBuilder.from((int) logQueryVO.getPage().getCurrent() * logQueryVO.getSize());
        sourceBuilder.size(logQueryVO.getSize());
        //6. 排序
        sourceBuilder.sort("@timestamp", logQueryVO.getSort());
        //7. 添加查询条件构建器 SearchSourceBuilder
        searchRequest.source(sourceBuilder);
        SearchResponse searchResponse = client.client().search(searchRequest, RequestOptions.DEFAULT);
        //8. 获取命中对象 SearchHits
        SearchHit[] hits = searchResponse.getHits().getHits();
        int i = 0;
        Page<LogResponseVO> page = logQueryVO.getPage();
        List<LogResponseVO> list = new ArrayList();
        int size = logQueryVO.getHighlight().size();
        // 9. 组装返回对象
        for (SearchHit hit : hits) {
            Map<String, Object> logDetailMap = hit.getSourceAsMap();
            String ProName = (String) logDetailMap.get("message");
            LogResponseVO response = new LogResponseVO();
            response.setHid(hit.getId());
            response.setMessage(ProName);
            response.setTimestamp((String) logDetailMap.get("@timestamp"));
            if (size > 0) {
                String highLightStr = getLightMessages(ProName, logQueryVO.getHighlight());
                response.setHigtlight(highLightStr);
            }
            response.setId(++i);
            list.add(response);
        }
        //10. 返回分页
        page.setRecords(list);
        page.setTotal(searchResponse.getHits().getTotalHits());
        page.setCurrent(logQueryVO.getPage().getCurrent());
        page.setPages(logQueryVO.getPage().getPages());
        return page;
    }

    /**
     * 高亮处理
     */
    private String getLightMessages(String proName, List<HighlightVO> highlight) {
        for (HighlightVO vo : highlight) {
            if (StringUtils.isEmpty(vo.getColor()) && StringUtils.isEmpty(vo.getWord())){
                proName = proName.replaceAll(vo.getWord(), "<text style=\"color: " + vo.getColor() + ";\">" + vo.getWord() + "</text>");
            }
        }
        return proName;
    }

    /**
     * 查询条件
     */
    public BoolQueryBuilder getBoolQuery(LogQueryVO logQueryVO) {
        //1.构建boolQuery
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        //2.构建各个查询条件
        //2.1 查询message名称为:关键字
        if (!StringUtils.isEmpty(logQueryVO.getKeyword())) {
            MatchQueryBuilder messageQueryBuilder = QueryBuilders.matchQuery("message", logQueryVO.getKeyword());
            boolQuery.must(messageQueryBuilder);
        }
        //2.2. 查询：系统名称
        if (!StringUtils.isEmpty(logQueryVO.getSysName())) {
            TermQueryBuilder sysNameQueryBuilder = QueryBuilders.termQuery("service.name", logQueryVO.getSysName());
            boolQuery.filter(sysNameQueryBuilder);
        }

        //2.3. 查询：日志等级
        if (!StringUtils.isEmpty(logQueryVO.getLogLevel())) {
            TermQueryBuilder levelQueryBuilder = QueryBuilders.termQuery("log.level", InfoLevelEnum.getValue(logQueryVO.getLogLevel()));
            boolQuery.filter(levelQueryBuilder);
        }
        //2.4. 查询：时间范围包含
        if (!StringUtils.isEmpty(logQueryVO.getStartTime()) && !StringUtils.isEmpty(logQueryVO.getEndTime())) {
            //时间格式：2022-03-22T07:28:46.111Z
            RangeQueryBuilder rangequerybuilder = QueryBuilders
                    .rangeQuery("@timestamp")
                    .from(DateUtil.format(logQueryVO.getStartTime(), DATE_UTC_PATTERN))
                    .to(DateUtil.format(logQueryVO.getEndTime(), DATE_UTC_PATTERN));
            boolQuery.filter(rangequerybuilder);
        }

        return boolQuery;
    }

解决超过10000行报错

PUT /filebeat-*/_settings
{ "index.max_result_window" :"100000000"}

或者在filebeat里面加入配置
“index.max_result_window”: “100000000”,