部署elk

docker stop $(docker ps -q)
docker rm $(docker ps -aq)
docker rmi -f $(docker images -qa)

docker network create elastic
sysctl -w vm.max_map_count=262144

1部署kibana

docker run -d -it --name kib01 -p  5601:5601 kibana:8.3.3

RequestOptions.DEFAULT替换
 HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory consumerFactory =
                new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(1024*1024*1024);
        RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder();
        builder.setHttpAsyncResponseConsumerFactory(consumerFactory);
        return builder.build();

rm -rf /home/elk/kibana/*
mkdir -p /home/elk/kibana/
docker cp kib01:/usr/share/kibana/config /home/elk/kibana/
docker cp kib01:/usr/share/kibana/data /home/elk/kibana/
docker cp kib01:/usr/share/kibana/logs /home/elk/kibana/
docker cp kib01:/usr/share/kibana/plugins /home/elk/kibana/
docker stop kib01
docker rm kib01

docker run --restart=always --privileged=true --name kib01 -p 5601:5601 \
-e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 \
--net elastic \
-v /home/elk/kibana/config:/usr/share/kibana/config \
-v /home/elk/kibana/data:/usr/share/kibana/data \
-v /home/elk/kibana/logs:/usr/share/kibana/logs \
-v /home/elk/kibana/plugins:/usr/share/kibana/plugins \
-d -it kibana:8.3.3

密码
kibana_system

2部署kibana

docker run  --name es01 --net elastic  -p 9200:9200 -p 9300:9300 -e ES_JAVA_OPTS="-Xms2048m -Xmx2048m" -e "discovery.type=single-node"  -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -it elasticsearch:8.3.3

rm -rf /home/elk/elasticsearch/*
mkdir -p /home/elk/elasticsearch/
docker cp es01:/usr/share/elasticsearch/config /home/elk/elasticsearch/
docker cp es01:/usr/share/elasticsearch/data /home/elk/elasticsearch/
docker cp es01:/usr/share/elasticsearch/logs /home/elk/elasticsearch/
docker cp es01:/usr/share/elasticsearch/plugins /home/elk/elasticsearch/
docker stop es01
docker rm es01

docker run --restart=always --privileged=true --name es01 -p 9200:9200 -p 9300:9300 \
--net elastic \
-e ES_JAVA_OPTS="-Xms2048m -Xmx2048m" -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -e "discovery.type=single-node" \
-v /home/elk/elasticsearch/config:/usr/share/elasticsearch/config \
-v /home/elk/elasticsearch/data:/usr/share/elasticsearch/data \
-v /home/elk/elasticsearch/logs:/usr/share/elasticsearch/logs \
-v /home/elk/elasticsearch/plugins:/usr/share/elasticsearch/plugins \
-dit elasticsearch:8.3.3

bin/elasticsearch-reset-password -u elastic
T0e*QUGWRt05*F-2PLFP

eyJ2ZXIiOiI4LjMuMyIsImFkciI6WyIxNzIuMTkuMC4zOjkyMDAiXSwiZmdyIjoiZWE4MjIxYTkyODUyODY3MjUyMDkwMTQyNWEyOTEyM2Q4NmM2ZDMwN2ZkZjk3NzgyNWRmMWE0NTkyZjRiYjg1MSIsImtleSI6IjNoTGVlWUlCallMdzg3OG1YcDlMOnBNbFNaMjlFU2ItX3NHX3ZtLTRsOWcifQ==

3、

docker exec -it kib01 /bin/sh 

cd bin 

sh kibana-verification-code

-- 输入code

docker exec -it es01 /bin/sh

4、创建es从节点
-- 创建令牌
docker exec -it es01 /usr/share/elasticsearch/bin/elasticsearch-create-enrollment-token -s node

docker run -e ES_JAVA_OPTS="-Xms256m -Xmx256m" \
-e ENROLLMENT_TOKEN="eyJ2ZXIiOiI4LjIuMCIsImFkciI6WyIxNzIuMTguMC4zOjkyMDAiXSwiZmdyIjoiOGUzZDY2MzJiZmJiM2JiNTc0N2FhNjNmMTc4ZTZjMGNiMzVlNjg5NGJhZjVjNjQ4Yzc1MjNlYzU4OTQ4YTMyYSIsImtleSI6InV5dWc4WUFCcGl4T0MySzNRRFVTOl82Wl9la2dtUS1XQk94RkY5QVJYakEifQ" \
--name es-member02 \
-p 19200:9200 \
--net elastic \
-dit elasticsearch:8.3.3

https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html

========================================================================================================================================================
# start the zookeeper service
bin/zookeeper-server-start.sh config/zookeeper.properties

# start the Kafka broker service
bin/kafka-server-start.sh config/server.properties

# create topic
bin/kafka-topics.sh --create --topic sparkKafkaEs --bootstrap-server localhost:9092

# describle message
bin/kafka-topics.sh --describe --topic sparkKafkaEs --bootstrap-server localhost:9092

# produce message
bin/kafka-console-producer.sh --topic sparkKafkaEs --bootstrap-server localhost:9092

# consumer message
bin/kafka-console-consumer.sh --topic sparkKafkaEs --from-beginning --bootstrap-server localhost:9092

问题记录：
 Cannot detect ES version - typically this happens if the network/Elasticsearch cluster is not accessible or when targeting a WAN/Cloud instance without the proper setting 'es.nodes.wan.only'
 增加配置
    conf.set("es.internal.es.cluster.name","docker-cluster")
    conf.set("es.internal.es.cluster.uuid","i033q5N4QuqsPAJs9-nCKQ")
    conf.set("es.internal.es.version","8.3.3")

 cannot assign instance of java.lang.invoke.SerializedLambda to field org.apache.spark.rdd.MapPartitionsRDD.f of type scala.Function3 in instance of org.apache.spark.rdd.MapPartitionsRDD
 
 
 
 
问题3、
sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: Path does not chain with any of the trust anchors); no other nodes left - aborting...

kubectl port-forward svc/ui 8080:8080 --namespace default