docker下部署elk用到的脚本

最新推荐文章于 2024-06-21 14:27:14 发布

weixin_32822759

最新推荐文章于 2024-06-21 14:27:14 发布

阅读量141

点赞数

文章标签： docker elk java

本文链接：https://blog.csdn.net/weixin_32822759/article/details/130924761

版权

1、docker-compose.yml

version: '2.2'
services:
  zookeeper:
    image: zookeeper:latest
    container_name: zookeper
    ports:
      - "2181:2181"                 
  kafka:
    image: wurstmeister/kafka:latest 
    container_name: kafka
    volumes: 
        - /etc/localtime:/etc/localtime 
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: 10.30.0.12   ## kafka所在的宿主机的ip
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181       
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_LOG_RETENTION_HOURS: 120
      KAFKA_MESSAGE_MAX_BYTES: 10000000
      KAFKA_REPLICA_FETCH_MAX_BYTES: 10000000
      KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS: 60000
      KAFKA_NUM_PARTITIONS: 3
      KAFKA_DELETE_RETENTION_MS: 1000                
  elasticsearch:
    image: elasticsearch:7.8.0   ## 此版本可以按需提升
    restart: always
    container_name: es
    environment:
     - discovery.type=single-node #单点启动，实际生产不允许
     - "ES_JAVA_OPTS=-Xms2048m -Xmx2048m"
     - "ELASTIC_PASSWORD=elastic"  ## 配置es的登录密码
    ports:
    - 9200:9200
    volumes:
       # 这里注意一下  如果你想吧docker中es的数据 映射出来 你本地的 /home/elasticsearch 必须拥有 777权限
      - /home/elk/elasticsearch/:/usr/share/elasticsearch/data
      - /home/elk/elasticsearch/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
  kibana:
    image: kibana:7.8.0   ## 此版本可以按需提升
    restart: always
    container_name: kibana
    ports:
    - 5601:5601
    volumes:
      ###将本机目录/home/elk/kibana/kibana.yml下的文件映射到docker容器里面
      - /home/elk/kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
    depends_on:
      - elasticsearch
  logstash:
    image: logstash:7.8.0   ## 此版本可以按需提升
    volumes:
        -  /home/elk/logstash/pipeline/:/usr/share/logstash/pipeline/
        -  /home/elk/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml 
        -  /home/elk/logstash/config/pipelines.yml:/usr/share/logstash/config/pipelines.yml 
    restart: always
    container_name: logstash
    ports:
    - 9600:9600
    depends_on:
      - elasticsearch

2、/home/elk/elasticsearch/elasticsearch.yml

## Default Elasticsearch configuration from Elasticsearch base image.
## https://github.com/elastic/elasticsearch/blob/master/distribution/docker/src/docker/config/elasticsearch.yml
#
cluster.name: "docker-cluster"
network.host: 0.0.0.0

## X-Pack settings
## see https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-xpack.html
#
xpack.license.self_generated.type: trial
xpack.security.enabled: true
xpack.monitoring.collection.enabled: true

3、/home/elk/kibana/kibana.yml

## Default Kibana configuration from Kibana base image.
### https://github.com/elastic/kibana/blob/master/src/dev/build/tasks/os_packages/docker_generator/templates/kibana_yml.template.js
server.name: kibana
server.host: 0.0.0.0
##汉化kibana 这个不能写在最下面 具体原因不清楚  写在最下面没作业 
i18n.locale: "zh-CN"
elasticsearch.hosts: [ "http://10.30.0.12:9200" ]
#elasticsearch 这里写的是你的ip
monitoring.ui.container.elasticsearch.enabled: true
#
### X-Pack security credentials
##
elasticsearch.username: elastic
elasticsearch.password: elastic

4、/home/elk/logstash/config/logstash.yml

## Default Logstash configuration from Logstash base image.
### https://github.com/elastic/logstash/blob/master/docker/data/logstash/config/logstash-full.yml
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://10.30.0.12:9200" ]   ## 配置es的主机地址，ip需要更换
#elasticsearch 这里写的是你的ip
## X-Pack security credentials
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.username: elastic
xpack.monitoring.elasticsearch.password: elastic

5、/home/elk/logstash/config/pipelines.yml

# List of pipelines to be loaded by Logstash
#
# This document must be a list of dictionaries/hashes, where the keys/values are pipeline settings.
# Default values for omitted settings are read from the `logstash.yml` file.
# When declaring multiple pipelines, each MUST have its own `pipeline.id`.
#
# Example of two pipelines:
  - pipeline.id: kafka
    pipeline.workers: 2 #线程数默认与cpu核数一致
    pipeline.batch.size: 1 #批量处理的条数默认125
    path.config: "/usr/share/logstash/pipeline"

6、/home/elk/logstash/pipeline/logstash-kafka.conf

input {
  kafka {
    bootstrap_servers => "10.30.0.12:9092"   # kafka所在的宿主机的ip地址，根据实际的ip，进行更换
    topics => ["tst-store-center"]   ## 发送消息时的主题
 }
}
filter {
  #Only matched data are send to output
}
output  {
  elasticsearch{
    hosts => ["10.30.0.12:9200"] # es所在的宿主机的ip地址，根据实际的ip，进行更换
    index => "tst-log-%{+YYYY-MM-dd}"  #  指定索引的格式
    user => "elastic"
    password => "elastic"
  }
  stdout{
    codec => rubydebug
  }
}

7、springboot中kafka的配置
yml文件中的配置如下：

kafka:
  # 生产者配置
  producer:
    servers: 10.30.0.12:9092
    retries: 0
    batchSize: 4096
    linger: 1
    bufferMemory: 1048576

  # 消费者配置
  consumer:
    servers: 10.30.0.12:9092
    concurrency: 3
    enableAutoCommit: false
    sessionTimeout: 6000

java代码配置

package com.tst.config;

import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.springframework.kafka.support.ProducerListener;
import org.springframework.stereotype.Component;

@Slf4j
@Component
public class KafkaSenderListener implements ProducerListener {

    public void onSuccess(String topic, Integer partition, Object key, Object value, RecordMetadata recordMetadata) {
        log.info(String.format(
                "生产者监听器，主题：%s，分区：%d，偏移量：%d，" + "key：%s，value：%s",
                recordMetadata.topic(),recordMetadata.partition(),
                recordMetadata.offset(),key,value));
    }

    public void onError(String topic, Integer partition, Object key, Object value, Exception exception) {
        exception.printStackTrace();
    }

    public boolean isInterestedInSuccess() {
        return true;
    }
}


package com.tst.config;

import com.tst.property.KafkaProducerProperty;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringSerializer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.core.DefaultKafkaProducerFactory;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.core.ProducerFactory;

import java.util.HashMap;
import java.util.Map;

@Configuration
@EnableKafka
public class KafkaProducerConfig {

    @Autowired
    private KafkaProducerProperty producerProperty;


    @Autowired
    private KafkaSenderListener senderListener;

    public Map<String, Object> producerConfigs() {
        Map<String, Object> props = new HashMap<>();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, producerProperty.getServers());
        props.put(ProducerConfig.RETRIES_CONFIG, producerProperty.getRetries());
        props.put(ProducerConfig.BATCH_SIZE_CONFIG, producerProperty.getBatchSize());
        // 等所有的副本都确认ok,才认为发送成功
        props.put(ProducerConfig.ACKS_CONFIG, "all");
        props.put(ProducerConfig.LINGER_MS_CONFIG, producerProperty.getLinger());
        props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, producerProperty.getBufferMemory());
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, 30000);
        return props;
    }

    public ProducerFactory<String, String> producerFactory() {
        return new DefaultKafkaProducerFactory<>(producerConfigs());
    }

    @Bean
    public KafkaTemplate kafkaTemplate() {
        KafkaTemplate kafkaTemplate = new KafkaTemplate(producerFactory()) ;
        kafkaTemplate.setProducerListener(senderListener); // 可以使用这种方式指定发送者监听器
        return kafkaTemplate;
    }
}

package com.tst.config;

import com.tst.property.KafkaConsumerProperty;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.kafka.annotation.EnableKafka;
import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory;
import org.springframework.kafka.config.KafkaListenerContainerFactory;
import org.springframework.kafka.core.ConsumerFactory;
import org.springframework.kafka.core.DefaultKafkaConsumerFactory;
import org.springframework.kafka.listener.ConcurrentMessageListenerContainer;
import org.springframework.kafka.listener.ContainerProperties;
import java.util.HashMap;
import java.util.Map;


@Configuration
@EnableKafka
public class KafkaConsumerConfig {

    @Autowired
    private KafkaConsumerProperty consumerProperty;

    // =======================================================手动确认模式的配置 开始==============================================================
    public Map<String, Object> consumerConfigsAck() {
        Map<String, Object> propsMap = new HashMap<>();
        propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, consumerProperty.getServers());
        propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        return propsMap;
    }

    public ConsumerFactory<String, String> consumerFactoryAck() {
        return new DefaultKafkaConsumerFactory<>(consumerConfigsAck());
    }

    @Bean("listenerAck")
    public CustomListenerAck listenerAck() {
        return new CustomListenerAck();
    }

    @Bean("factoryAckContainer")
    public KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactoryAck() {
        ConcurrentKafkaListenerContainerFactory<String, String> factory = new ConcurrentKafkaListenerContainerFactory<>();
        factory.setConsumerFactory(consumerFactoryAck());
        factory.setConcurrency(consumerProperty.getConcurrency());
        factory.getContainerProperties().setPollTimeout(500);
        factory.getContainerProperties().setAckMode(ContainerProperties.AckMode.MANUAL_IMMEDIATE);
        return factory;
    }
}

package com.tst.config;

import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.kafka.support.Acknowledgment;

@Slf4j
public class CustomListenerAck {

//    @KafkaListener(groupId = "business-log", topics = {"tst-store-center"},containerFactory = "factoryAckContainer")
    public void listen(ConsumerRecord<?, ?> record, Acknowledgment ack) {
        try {
            log.info("自行确认方式收到消息的key: " + record.key());
            log.info("自行确认方式收到消息的value: " + record.value().toString());

            // todo 做具体的业务

            log.info("消息确认！");
            ack.acknowledge();
        } catch (Exception e){
            ack.nack(10000);
        }
    }
}

package com.tst.config;

import com.tst.util.ExceptionUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Component;

@Component
public class KafkaSender {

    @Autowired
    private KafkaTemplate kafkaTemplate;

    public void messageSender(String tpoic,String key,Object message){
        try {
            kafkaTemplate.send(tpoic,key,message);
        } catch (Exception e) { kafkaTemplate.send(tpoic,key, ExceptionUtil.printExStack(e));
        }
    }
}

package com.tst.property;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;

@Data
@RefreshScope
@ConfigurationProperties(prefix = "kafka.producer")
public class KafkaProducerProperty {

    /**
     * 服务器地址
     */
    private String servers;

    /**
     * 重试次数
     */
    private Integer retries;

    private Integer batchSize;

    private Integer linger;

    private String bufferMemory;
}

package com.tst.property;

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.cloud.context.config.annotation.RefreshScope;

@Data
@RefreshScope
@ConfigurationProperties(prefix = "kafka.consumer")
public class KafkaConsumerProperty {

    /**
     * 服务器地址
     */
    private String servers;

    /**
     * 并发度
     */
    private Integer concurrency;

    private Boolean enableAutoCommit;

    private Integer sessionTimeout;
}