把日志灌入到Elasticsearch的好处以及具体实现

2019独角兽企业重金招聘Python工程师标准>>>

一般来讲一个高并发高性能的系统,日志是非常庞大的,随时可能高达几个T,一台服务器的硬盘极有可能装不下,而Elasticsearch的集群可以分布在不同的机器上,而又对整个集群作为一个整体,对其大容量的内容进行存储以及它的最牛掰的能力--检索.

首先在配置文件中做如下配置

elasticsearch:
  clusterName: aubin-cluster
  clusterNodes: 192.168.5.182:9300

@Getter
@Setter
@Configuration
@ConfigurationProperties(prefix = "elasticsearch")
public class ElasticSearchConfig {

   private String clusterName;

   private String clusterNodes;

    /**
     * 使用elasticsearch实现类时才触发
     *
     * @return
     */
   @Bean
    @ConditionalOnBean(value = EsLogServiceImpl.class)
   public TransportClient getESClient() {
      // 设置集群名字
      Settings settings = Settings.builder().put("cluster.name", this.clusterName).build();
      TransportClient client = new PreBuiltTransportClient(settings);
      try {
         // 读取的ip列表是以逗号分隔的
         for (String clusterNode : this.clusterNodes.split(",")) {
            String ip = clusterNode.split(":")[0];
            String port = clusterNode.split(":")[1];
            client.addTransportAddress(new TransportAddress(InetAddress.getByName(ip), Integer.parseInt(port)));
         }
      } catch (UnknownHostException e) {
         e.printStackTrace();
      }

      return client;
   }
}

再定义一个操作接口

public interface LogService {

   void save(Log log);

   Page<Log> findLogs(Map<String, Object> params);

}

具体实现类为

@Service
public class EsLogServiceImpl implements LogService, ApplicationContextAware {

   private static final Logger logger = LoggerFactory.getLogger(EsLogServiceImpl.class);

   private static final String INDEX = "index_logs";
   private static final String TYPE = "type_logs";

   @Autowired
   private TransportClient client;

   @Async
   @Override
   public void save(Log log) {
      if (log.getCreateTime() == null) {
         log.setCreateTime(new Date());
      }
      if (log.getFlag() == null) {
         log.setFlag(Boolean.TRUE);
      }
      logger.info("{}", log);

      String string = JSONObject.toJSONString(log);

      IndexRequestBuilder builder = client.prepareIndex(INDEX, TYPE).setSource(string, XContentType.JSON);
      builder.execute();
   }

   @Override
   public Page<Log> findLogs(Map<String, Object> params) {
      SearchRequestBuilder builder = client.prepareSearch().setIndices(INDEX).setTypes(TYPE);
      if (!CollectionUtils.isEmpty(params)) {
         BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();

         // 用户名模糊匹配
         String username = MapUtils.getString(params, "username");
         if (StringUtils.isNoneBlank(username)) {
            queryBuilder.must(QueryBuilders.wildcardQuery("username", "*" + username + "*"));
         }

         // 模块精确匹配
         String module = MapUtils.getString(params, "module");
         if (StringUtils.isNoneBlank(module)) {
            queryBuilder.must(QueryBuilders.matchQuery("module", module));
         }

         String flag = MapUtils.getString(params, "flag");
         if (StringUtils.isNoneBlank(flag)) {
            Boolean bool = Boolean.FALSE;
            if ("1".equals(flag) || "true".equalsIgnoreCase(flag)) {
               bool = Boolean.TRUE;
            }
            queryBuilder.must(QueryBuilders.matchQuery("flag", bool));
         }

         // 大于等于开始日期,格式yyyy-MM-dd
         String beginTime = MapUtils.getString(params, "beginTime");
         if (StringUtils.isNoneBlank(beginTime)) {
            // 转化为0点0分0秒
            Long timestamp = toTimestamp(beginTime + "T00:00:00");
            queryBuilder.must(QueryBuilders.rangeQuery("createTime").from(timestamp));
         }

         // 小于等于结束日期,格式yyyy-MM-dd
         String endTime = MapUtils.getString(params, "endTime");
         if (StringUtils.isNoneBlank(endTime)) {
            // 转化为23点59分59秒
            Long timestamp = toTimestamp(endTime + "T23:59:59");
            queryBuilder.must(QueryBuilders.rangeQuery("createTime").to(timestamp));
         }

         if (queryBuilder != null) {
            builder.setPostFilter(queryBuilder);
         }
      }

      builder.addSort("createTime", SortOrder.DESC);

      PageUtil.pageParamConver(params, true);
      Integer start = MapUtils.getInteger(params, PageUtil.START);
      if (start != null) {
         builder.setFrom(start);
      }

      Integer length = MapUtils.getInteger(params, PageUtil.LENGTH);
      if (length != null) {
         builder.setSize(length);
      }

      SearchResponse searchResponse = builder.get();

      SearchHits searchHits = searchResponse.getHits();
      // 总数量
      Long total = searchHits.getTotalHits();

      int size = searchHits.getHits().length;
      List<Log> list = new ArrayList<>(size);
      if (size > 0) {
         searchHits.forEach(hit -> {
            String val = hit.getSourceAsString();
            list.add(JSONObject.parseObject(val, Log.class));
         });
      }

      return new Page<>(total.intValue(), list);
   }

   private Long toTimestamp(String str) {
      LocalDateTime localDateTime = LocalDateTime.parse(str);
      Date date = Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant());

      return date.getTime();
   }

   private static ApplicationContext applicationContext = null;

   @Override
   public void setApplicationContext(ApplicationContext context) throws BeansException {
      applicationContext = context;
   }

   /**
    * 初始化日志es索引
    */
   @PostConstruct
   public void initIndex() {
      LogService logService = applicationContext.getBean(LogService.class);
      // 日志实现是否采用elasticsearch
      boolean flag = (logService instanceof EsLogServiceImpl);
      if (!flag) {
         return;
      }

      try {
         // 判断索引是否存在
         IndicesExistsResponse indicesExistsResponse = client.admin().indices()
               .exists(new IndicesExistsRequest(INDEX)).get();
         if (indicesExistsResponse.isExists()) {
            return;
         }
      } catch (InterruptedException e) {
         e.printStackTrace();
      } catch (ExecutionException e) {
         e.printStackTrace();
      }

      CreateIndexRequestBuilder requestBuilder = client.admin().indices().prepareCreate(INDEX);

      CreateIndexResponse createIndexResponse = requestBuilder.execute().actionGet();
      if (createIndexResponse.isAcknowledged()) {
         logger.info("索引：{},创建成功", INDEX);
      } else {
         logger.error("索引：{},创建失败", INDEX);
      }
   }

}

转载于:https://my.oschina.net/u/3768341/blog/1934681