ELK分布式日志收集

一、环境部署

es 要求Linux 的 参数 vm.max_map_count 至少为 262144，目录需要赋权限: chmod -R 777 search

新建docker-compose.yml

version: '3.1'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
    container_name: elasticsearch
    restart: always
    volumes:
      - ./search/es/esdata:/usr/share/elasticsearch/data
      #- ./search/config/es01/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - ./search/es/logs/elasticsearch:/usr/share/elasticsearch/logs
      - ./search/es/plugins/analysis-ik:/usr/share/elasticsearch/plugins/analysis-ik
    environment:
      - discovery.type=single-node
      - "ES_JAVA_OPTS=-Xms4g -Xmx4g"
      - TZ=Asia/Shanghai
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - elk

  kibana:
    image: docker.elastic.co/kibana/kibana:7.6.2
    links:
      - elasticsearch
    environment:
      ELASTICSEARCH_HOSTS: http://elasticsearch:9200
    volumes:
      - ./kibana_config/:/usr/local/kibana/config/
    ports:
      - "5601:5601"
    depends_on:
      - elasticsearch 
    networks:
      - elk
    container_name: kibana
    restart: always

  logstash:
    image: docker.elastic.co/logstash/logstash:7.6.2
    links:
      - elasticsearch
    command: logstash -f /etc/logstash/config/logstash.conf  #logstash 启动时使用的配置文件
    volumes:
      - ./search/logstash/config/logstash.conf:/etc/logstash/config/logstash.conf  #logstash 配文件位置
      - ./search/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml  #logstash 配文件位置
     # - ./search/logstash/log/pv.log:/logs/pv.log #挂载要读取日志文件
    depends_on:
      - elasticsearch  #后于elasticsearch启动
    ports:
      - "4560:4560"
      # - "8065:8065"
      # - "9600:9600"
    networks:
      - elk 
    container_name: logstash

networks:
  elk:

新建./search/es/plugins/analysis-ik安装ik分词器

https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.6.2/elasticsearch-analysis-ik-7.6.2.zip

新建./search/logstash/config/logstash.conf

input {
	tcp {
		mode => "server"
		port => 4560
		codec => json #json格式读取
	}
}
filter {
	json{
		source => "message"
		remove_field => ["host","port","message","@version"] #去掉不要的字段
  }
}

output {
	elasticsearch {
		action => "index"
		hosts  => "172.18.42.142:9200"
		index  => "test_log"
	}
}

新建./search/logstash/config/logstash.yml

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: ["http://172.0.0.1:9200"] #设置es地址

二、与项目整合

springboot微服务的pom文件添加

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
    <groupId>net.logstash.logback</groupId>
    <artifactId>logstash-logback-encoder</artifactId>
    <version>6.6</version>
</dependency>

resources/application.yml加入

spring:
    elasticsearch:
        rest:
          uris: http://172.0.0.1:9200 #es地址

resources/logback.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--该日志将日志级别不同的log信息保存到不同的文件中-->
<configuration>
	<include resource="org/springframework/boot/logging/logback/defaults.xml" />

<!--	<springProperty scope="context" name="destination" source="net.logstash.server.hosts"  />-->
	<springProperty scope="context" name="springAppName" source="spring.application.name"  />

	<!-- 控制台输出	-->
	<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
		<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
			<level>INFO</level>
		</filter>
		<!-- 日志输出编码 -->
		<encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
			<providers class="net.logstash.logback.composite.loggingevent.LoggingEventJsonProviders">
				<pattern>
					<pattern>
						{
						"date":"%date{\"yyyy-MM-dd HH:mm:ss.SSS\",UTC}",
						"level":"%level",
						"msg":"%msg"
						}
					</pattern>
				</pattern>
			</providers>
		</encoder>
	</appender>

	<appender name="LOGSTASH" class="net.logstash.logback.appender.LogstashTcpSocketAppender">
		<destination>172.0.0.1:4560</destination>
		<encoder charset="UTF-8"
				 class="net.logstash.logback.encoder.LogstashEncoder">
		</encoder>
	</appender>

	<!--异步日志-->
	<appender name="async" class="ch.qos.logback.classic.AsyncAppender">
		<!--指定摸个具体的 appender-->
		<appender-ref ref="LOGSTASH" />
	</appender>

	<root level="INFO">
<!--		<appender-ref ref="LOGSTASH" />-->
		<appender-ref ref="CONSOLE" />
	</root>

	<!-- 自定义 logger 对象
	name="com.demo.weblog" 设置传入logstash的日志
	additivity="false" 自定义 logger 对象是否继承 rootlogger
	-->
	<logger name="com.demo.weblog" level="info" additivity="false">
	    <appender-ref ref="async" />
	</logger>

</configuration>

es的整合

# domain/WebLog
@Data
@Document(indexName = "test_log",shards = 1,replicas = 1)
public class WebLog {

    @Id
    private String id;

    @Field(type = FieldType.Integer)
    private Integer port;

    @Field(type = FieldType.Text)
    private String message;

    @Field(name = "@version",type = FieldType.Keyword)
    private String version;

    @Field(name = "@timestamp", type = FieldType.Date,format = DateFormat.date_time)
    private Date timestamp;

    @Field(type = FieldType.Keyword)
    private String host;

    private LogMessage lm;
}

# repository/TestRepository
public interface TestRepository extends ElasticsearchRepository<WebLog,String> {
}

# controller/WebLogController
@Api(tags = "es信息接口")
@RestController
public class WebLogController {
    @Autowired
    private TestRepository testRepository;
    
    @ApiOperation(value = "test详情")
    @GetMapping("/testfindById/{id}")
    public R<Optional<WebLog>> testfindById(@PathVariable String id) {
        return R.ok(testRepository.findById(id));
    }
}

三、esAPI用法

pom.xml配置

<properties>
    <java.version>1.8</java.version>
    <elasticsearch.version>7.6.2</elasticsearch.version>
</properties>
<dependency>
   <groupId>org.elasticsearch.client</groupId>
   <artifactId>elasticsearch-rest-high-level-client</artifactId>
   <version>${elasticsearch.version}</version>
</dependency>

测试代码

public static void main(String[] args) throws IOException {
        RestHighLevelClient client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost("172.0.0.1", 9200, "http")));

        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        TermQueryBuilder termQueryBuilder  = QueryBuilders.termQuery("serviceName", "11111");
        sourceBuilder.query(termQueryBuilder);//精确
//        sourceBuilder.from(0);
        sourceBuilder.size(1);
//        sourceBuilder.collapse(new CollapseBuilder("serviceName.keyword"));
//        QueryBuilders.rangeQuery("requestTime").gte("2021-10-27T15:38:00").lte("2021-10-27T15:39:00");
//        sourceBuilder.query(QueryBuilders.matchAllQuery());
//        AggregationBuilder aggregationBuilder = AggregationBuilders.dateHistogram("by_time").field("requestTime").fixedInterval(DateHistogramInterval.hours(5));
//        SumAggregationBuilder aggregation =
//                AggregationBuilders
//                        .sum("timeConsuming")
//                        .field("timeConsuming");
//        aggregationBuilder.subAggregation(aggregation);
//        sourceBuilder.aggregation(aggregationBuilder);
        // 2.创建并设置SearchRequest对象
        // 设置request要搜索的索引和类型
        SearchRequest searchRequest = new SearchRequest("test_log");
        // 设置SearchSourceBuilder查询属性
        sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        searchRequest.source(sourceBuilder);


        // 3.查询
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
        log.info(JSON.toJSONString(searchResponse.getHits(), true));
        System.out.println("==============================");
        for (SearchHit documentFields : searchResponse.getHits().getHits()){
            System.out.println(documentFields.getSourceAsMap().get("status"));
        }

        client.close();

    }

四、es问题修复

1，text转keyword

PUT /test_log/_mapping?include_type_name=false
{
  "properties": {
    "type": { 
      "ignore_above": 10, //大于10个字符将不进行索引，即无法检索
      "type":     "keyword"
    }
  }
}