简介
Zookeeper集群;(必须事先准备);
1、介绍kafka是什么;
消息队列;kafka就是一个消息队列MQ; elk需要使用kafka来传递日志消息;
一、传统方式部署kafka集群
1 环境说明
192.168.79.34 node1
192.168.79.35 node2
192.168.79.36 node3
#仍在之前的kafka环境
2 node节点操作:
#所有节点操作:
yum install java java-devel -y
wget https://archive.apache.org/dist/kafka/2.2.0/kafka_2.12-2.2.0.tgz
tar xf kafka_2.12-2.2.0.tgz -C /opt
ln -s /opt/kafka_2.12-2.2.0/ /opt/kafka
mkdir /opt/kafka/data
3 修改kafka配置
cat /opt/kafka/config/server.properties
node1
############################# Server Basics
# broker的id,值为整数,且必须唯一,在一个集群中不能重复
broker.id=1
############################# Socket ServerSettings
# kafka监听端口,默认9092
listeners=PLAINTEXT://192.168.79.34:9092
# 处理网络请求的线程数量,默认为3个
num.network.threads=3
# 执行磁盘IO操作的线程数量,默认为8个
num.io.threads=8
# socket服务发送数据的缓冲区大小,默认100KB
socket.send.buffer.bytes=102400
# socket服务接受数据的缓冲区大小,默认100KB
socket.receive.buffer.bytes=102400
# socket服务所能接受的一个请求的最大大小,默认为100M
socket.request.max.bytes=104857600
############################# Log Basics
# kafka存储消息数据的目录
log.dirs=../data
# 每个topic默认的partition
num.partitions=1
# 设置副本数量为3,当Leader的Replication故障,会进行故障自动转移。
default.replication.factor=3
# 在启动时恢复数据和关闭时刷新数据时每个数据目录的线程数量
num.recovery.threads.per.data.dir=1
############################# Log Flush Policy
# 消息刷新到磁盘中的消息条数阈值
log.flush.interval.messages=10000
# 消息刷新到磁盘中的最大时间间隔,1s
log.flush.interval.ms=1000
############################# Log Retention Policy
# 日志保留小时数,超时会自动删除,默认为7天
log.retention.hours=168
# 日志保留大小,超出大小会自动删除,默认为1G
#log.retention.bytes=1073741824
# 日志分片策略,单个日志文件的大小最大为1G,超出后则创建一个新的日志文件
log.segment.bytes=1073741824
# 每隔多长时间检测数据是否达到删除条件,300s
log.retention.check.interval.ms=300000
############################# Zookeeper
# Zookeeper连接信息,如果是zookeeper集群,则以逗号隔开
zookeeper.connect=192.168.79.34:2181,192.168.79.35:2181,192.168.79.36:2181
# 连接zookeeper的超时时间,6s
zookeeper.connection.timeout.ms=6000
node2
############################# Server Basics
# broker的id,值为整数,且必须唯一,在一个集群中不能重复
broker.id=2
############################# Socket ServerSettings
# kafka监听端口,默认9092
listeners=PLAINTEXT://192.168.79.35:9092
# 处理网络请求的线程数量,默认为3个
num.network.threads=3
# 执行磁盘IO操作的线程数量,默认为8个
num.io.threads=8
# socket服务发送数据的缓冲区大小,默认100KB
socket.send.buffer.bytes=102400
# socket服务接受数据的缓冲区大小,默认100KB
socket.receive.buffer.bytes=102400
# socket服务所能接受的一个请求的最大大小,默认为100M
socket.request.max.bytes=104857600
############################# Log Basics
# kafka存储消息数据的目录
log.dirs=../data
# 每个topic默认的partition
num.partitions=1
# 设置副本数量为3,当Leader的Replication故障,会进行故障自动转移。
default.replication.factor=3
# 在启动时恢复数据和关闭时刷新数据时每个数据目录的线程数量
num.recovery.threads.per.data.dir=1
############################# Log Flush Policy
# 消息刷新到磁盘中的消息条数阈值
log.flush.interval.messages=10000
# 消息刷新到磁盘中的最大时间间隔,1s
log.flush.interval.ms=1000
############################# Log Retention Policy
# 日志保留小时数,超时会自动删除,默认为7天
log.retention.hours=168
# 日志保留大小,超出大小会自动删除,默认为1G
#log.retention.bytes=1073741824
# 日志分片策略,单个日志文件的大小最大为1G,超出后则创建一个新的日志文件
log.segment.bytes=1073741824
# 每隔多长时间检测数据是否达到删除条件,300s
log.retention.check.interval.ms=300000
############################# Zookeeper
# Zookeeper连接信息,如果是zookeeper集群,则以逗号隔开
zookeeper.connect=192.168.79.34:2181,192.168.79.35:2181,192.168.79.36:2181
# 连接zookeeper的超时时间,6s
zookeeper.connection.timeout.ms=6000
node3
############################# Server Basics
# broker的id,值为整数,且必须唯一,在一个集群中不能重复
broker.id=3
############################# Socket ServerSettings
# kafka监听端口,默认9092
listeners=PLAINTEXT://192.168.79.36:9092
# 处理网络请求的线程数量,默认为3个
num.network.threads=3
# 执行磁盘IO操作的线程数量,默认为8个
num.io.threads=8
# socket服务发送数据的缓冲区大小,默认100KB
socket.send.buffer.bytes=102400
# socket服务接受数据的缓冲区大小,默认100KB
socket.receive.buffer.bytes=102400
# socket服务所能接受的一个请求的最大大小,默认为100M
socket.request.max.bytes=104857600
############################# Log Basics
# kafka存储消息数据的目录
log.dirs=../data
# 每个topic默认的partition
num.partitions=1
# 设置副本数量为3,当Leader的Replication故障,会进行故障自动转移。
default.replication.factor=3
# 在启动时恢复数据和关闭时刷新数据时每个数据目录的线程数量
num.recovery.threads.per.data.dir=1
############################# Log Flush Policy
# 消息刷新到磁盘中的消息条数阈值
log.flush.interval.messages=10000
# 消息刷新到磁盘中的最大时间间隔,1s
log.flush.interval.ms=1000
############################# Log Retention Policy
# 日志保留小时数,超时会自动删除,默认为7天
log.retention.hours=168
# 日志保留大小,超出大小会自动删除,默认为1G
#log.retention.bytes=1073741824
# 日志分片策略,单个日志文件的大小最大为1G,超出后则创建一个新的日志文件
log.segment.bytes=1073741824
# 每隔多长时间检测数据是否达到删除条件,300s
log.retention.check.interval.ms=300000
############################# Zookeeper
# Zookeeper连接信息,如果是zookeeper集群,则以逗号隔开
zookeeper.connect=192.168.79.34:2181,192.168.79.35:2181,192.168.79.36:2181
# 连接zookeeper的超时时间,6s
zookeeper.connection.timeout.ms=6000
4 启动kafka
cd /opt/kafka/bin
./kafka-server-start.sh -daemon ../config/server.properties
5 kafka集群验证
1、使用kafka创建一个top
cd /opt/kafka/bin
./kafka-topics.sh --create --zookeeper 192.168.79.34:2181,192.168.79.35:2181,192.168.79.36:2181 --partitions 1 --replication-factor 3 --topic lss
2、模拟消息发布者
cd /opt/kafka/bin
# ./kafka-console-producer.sh --broker-list 192.168.79.34:9092,192.168.79.35:9092,192.168.79.36:9092 --topic lss
[root@node4 bin]# ./kafka-console-producer.sh \
> --broker-list 192.168.79.34:9092,192.168.79.35:9092,192.168.79.36:9092 \
> --topic lss
>hello lss
>hello kafka
>
>hellp enchanted
>
3、模拟消息订阅者
cd /opt/kafka/bin
#./kafka-console-consumer.sh --bootstrap-server 192.168.79.34:9092,192.168.79.35:9092,192.168.79.36:9092 --topic lss --from-beginning
[root@node4 bin]# ./kafka-console-consumer.sh \
> --bootstrap-server 192.168.79.34:9092,192.168.79.35:9092,192.168.79.36:9092 \
> --topic lss \
> --from-beginning
hello lss
hello kafka
hellp enchanted
二、传统方式部署efak (kafka可视化)
安装配置jdk (略)
1 efak安装
wget https://linux.oldxu.net/efak-web-3.0.1-bin.tar.gz
tar xf efak-web-3.0.1-bin.tar.gz -C /opt/
ln -s /opt/efak-web-3.0.1/ /opt/efak-web
vim /etc/profile
export KE_HOME=/opt/efak
export PATH=$PATH:$KE_HOME/bin
source /etc/profile
3.1 efak配置修改
cat /opt/efak-web/conf/system-config.properties
# 填写 zookeeper集群列表
efak.zk.cluster.alias=cluster1
cluster1.zk.list=192.168.79.34:2181,192.168.79.35:2181,192.168.79.36:2181
# broker 最大规模数量
cluster1.efak.broker.size=20
# zk 客户端线程数
kafka.zk.limit.size=32
# EFAK webui 端口
efak.webui.port=8048
# kafka offset storage
cluster1.efak.offset.storage=kafka
# kafka jmx uri
cluster1.efak.jmx.uri=service:jmx:rmi://jndi/rmi://%s/jmxrmi
# kafka metrics 指标,默认存储15天
efak.metrics.charts=true
efak.metrics.retain=15
# kafka sql topic records max
efak.sql.topic.records.max=5000
efak.sql.topic.preview.records.max=10
# delete kafka topic token
efak.topic.token=keadmin
# kafka sqlite 数据库地址(需要修改存储路径)
efak.driver=org.sqlite.JDBC
efak.url=jdbc:sqlite:/opt/efak/db/ke.db
efak.username=root
efak.password=www.kafka-eagle.org
# kafka mysql 数据库地址(需要提前创建ke库)
efak.driver=com.mysql.cj.jdbc.Driver
efak.url=jdbc:mysql://192.168.79.35:3306/ke?useUnicode=true&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull
efak.username=ke
efak.password=123456
3.2 配置数据库
#79.35操作
mysql -uroot -p123456 -e "create database ke"
mysql> set global validate_password_policy=0;
mysql> set global validate_password_length=1;
mysql> create user 'ke'@'%' identified by '123456';
mysql> grant all privileges on *.* to 'ke'@'192.168.79.%' IDENTIFIED BY '123456';
mysql> flush privileges;
4 启动efak
/opt/efak-web/bin/ke.sh start
5 访问efak (admin / 123456)
三、制作Kafka集群镜像
2.1 Dockerfile
FROM openjdk:8-jre
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
echo 'Asia/Shanghai' > /etc/timezone
ENV VERSION=2.12-2.2.0
ADD ./kafka_${VERSION}.tgz /
ADD ./server.properties /kafka_${VERSION}/config/server.properties
RUN mv /kafka_${VERSION} /kafka
ADD ./entrypoint.sh /entrypoint.sh
EXPOSE 9092 9999
CMD ["bin/bash","/entrypoint.sh"]
2.2 server.properties
# Server Basics
# broker的id,值为整数,且必须唯一,在一个集群中不能重复
broker.id={BROKER_ID}
# Socket Server Settings
# kafka监听端口,默认9092
listeners=PLAINTEXT://{LISTENERS}:9092
# 处理网络请求的线程数量,默认为3个
num.network.threads=3
# 执行磁盘IO操作的线程数量,默认为8个
num.io.threads=8
# socket服务发送数据的缓冲区大小,默认100KB
socket.send.buffer.bytes=102400
# socket服务接受数据的缓冲区大小,默认100KB
socket.receive.buffer.bytes=102400
# socket服务所能接受的一个请求的最大大小,默认为100M
socket.request.max.bytes=104857600
# Log Basics
# kafka存储消息数据的目录
log.dirs={KAFKA_DATA_DIR}
# 每个topic默认的partition
num.partitions=1
# 设置副本数量为3,当Leader的Replication故障,会进行故障自动转移。
default.replication.factor=3
# 在启动时恢复数据和关闭时刷新数据时每个数据目录的线程数量
num.recovery.threads.per.data.dir=1
## Log Flush Policy
# 消息刷新到磁盘中的消息条数阈值
log.flush.interval.messages=10000
# 消息刷新到磁盘中的最大时间间隔,1s
log.flush.interval.ms=1000
## Log Retention Policy
# 日志保留小时数,超时会自动删除,默认为7天
log.retention.hours=168
# 日志保留大小,超出大小会自动删除,默认为1G
log.segment.bytes=1073741824
# 每隔多长时间检测数据是否达到删除条件,300s
log.retention.check.interval.ms=300000
# Zookeeper
# Zookeeper连接信息,如果是zookeeper集群,则以逗号隔开
zookeeper.connect={ZOOK_SERVERS}
# 连接zookeeper的超时时间,6s
zookeeper.connection.timeout.ms=6000
2.3 entrypoint.sh
# 需要替换的内容
# {BROKER_ID}、{LISTENERS}、{KAFKA_DATA_DIR}、{ZOOK_SERVERS}
KAFKA_DIR=/kafka
KAFKA_CONF=/kafka/config/server.properties
BROKER_ID=$(( $(hostname | sed 's#.*-##g') + 1 ))
LISTENERS=$(hostname -i)
#替换配置内容
sed -i s@{BROKER_ID}@${BROKER_ID}@g ${KAFKA_CONF}
sed -i s@{LISTENERS}@${LISTENERS}@g ${KAFKA_CONF}
sed -i s@{ZOOK_SERVERS}@${ZOOK_SERVERS}@g ${KAFKA_CONF}
sed -i s@{KAFKA_DATA_DIR}@${KAFKA_DATA_DIR:-/data}@g ${KAFKA_CONF}
# 启动kafka,并kafka启动脚本添加JMX端口
cd $KAFKA_DIR/bin
sed -i '/export KAFKA_HEAP_OPTS/a export JMX_PORT="9999"' kafka-server-start.sh
./kafka-server-start.sh ../config/server.properties
2.4 构建镜像并推送仓库
wget https://archive.apache.org/dist/kafka/2.2.0/kafka_2.12-2.2.0.tgz
[root@node4 kafka-dockerfile]# ls
Dockerfile entrypoint.sh kafka_2.12-2.2.0.tgz server.properties
docker build -t harbor.oldxu.net/base/kafka:2.12.2 .
docker push harbor.oldxu.net/base/kafka:2.12.2
四、迁移kafka集群到k8s
01-kafka-headless.yaml
apiVersion: v1
kind: Service
metadata:
name: kafka-svc
spec:
clusterIP: None
selector:
app: kafka
ports:
- name: client
port: 9092
targetPort: 9092
- name: jmx
port: 9999
targetPort: 9999
02-kafka-sts.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: kafka
spec:
serviceName: "kafka-svc"
replicas: 3
selector:
matchLabels:
app: kafka
template:
metadata:
labels:
app: kafka
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values: ["kafka"]
topologyKey: "kubernetes.io/hostname"
imagePullSecrets:
- name: harbor-login
containers:
- name: kafka
image: harbor.oldxu.net/base/kafka:2.12.2
imagePullPolicy: IfNotPresent
ports:
- name: client
containerPort: 9092
- name: jmxport
containerPort: 9999
env:
- name: ZOOK_SERVERS
value: "zookeeper-0.zk-svc:2181,zookeeper-1.zk.svc:2181,zookeeper-2.zk-svc:2181"
readinessProbe: #就绪探针,不就绪则不介入流量
tcpSocket:
port: 9092
initialDelaySeconds: 5
livenessProbe: #存活探针。如果不存活则根据重启策略进行重启
tcpSocket:
port: 9092
initialDelaySeconds: 5
volumeMounts:
- name: data
mountPath: /data
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes: ["ReadWriteMany"]
storageClassName: "nfs"
resources:
requests:
storage: 15Gi
验证kafka
[root@master01 kafka-efak-Project]# kubectl describe svc kafka-svc
Name: kafka-svc
Namespace: default
Labels: <none>
Annotations: Selector: app=kafka
Type: ClusterIP
IP: None
Port: client 9092/TCP
TargetPort: 9092/TCP
Endpoints: 10.244.0.35:9092,10.244.1.16:9092,10.244.2.241:9092
Port: jmx 9999/TCP
TargetPort: 9999/TCP
Endpoints: 10.244.0.35:9999,10.244.1.16:9999,10.244.2.241:9999
Session Affinity: None
Events: <none>
1 使用kafka命令创建一个topic
[root@master01 ~]# kubectl exec -it kafka-0 -- bash
root@kafka-0:/# hostname -f
kafka-0.kafka-svc.default.svc.cluster.local
root@kafka-0:/# /kafka/bin/kafka-topics.sh \
> --create \
> --zookeeper zookeeper-0.zk-svc:2181,zookeeper-1.zk-svc:2181,zookeeper-2.zk-svc:2181 \
> --partitions 1 \
> --replication-factor 3 \
> --topic lss
Created topic lss.
2、模拟消息发布者
[root@master01 ~]# kubectl exec -it kafka-1 -- bash
root@kafka-1:/# /kafka/bin/kafka-console-producer.sh --broker-list kafka-0.kafka-svc:9092,kafka-1.kafka-svc:9092,kafka-2.kafka-svc:9092 --topic lss
>hellp kafaka
>hellp lss
>hello lss
>hello crush
>bye lss
>
3、模拟消息订阅者
[root@master01 ~]# kubectl exec -it kafka-2 -- bash
root@kafka-2:/# /kafka/bin/kafka-console-consumer.sh \
> --bootstrap-server kafka-0.kafka-svc:9092,kafka-1.kafka-svc:9092,kafka-2.kafka-svc:9092 \
> --topic lss \
> --from-beginning
hellp kafaka
hellp lss
hello lss
hello crush
123456
bye lss
五、制作efak镜像
5.1 Dockerfile
FROM openjdk:8
# 1、调整时区
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
echo "Asia/Shanghai" > /etc/timezone
# 2、拷贝kafka软件以及kafka的配置
ENV VERSION=3.0.1
ADD ./efak-web-${VERSION}-bin.tar.gz /
ADD ./system-config.properties /efak-web-${VERSION}/conf/system-config.properties
# 3、修改kafka的名称
RUN mv /efak-web-${VERSION} /efak
# 4\环境变量
ENV KE_HOME=/efak
ENV PATH=$PATH:$KE_HOME/bin
# 4、启动脚本(修改kafka配置)
ADD ./entrypoint.sh /entrypoint.sh
# 5、暴露kafka端口 9999是jmx的端口
EXPOSE 8048
# 6、运行启动脚本
CMD ["/bin/bash","/entrypoint.sh"]
5.2 system-config.properties
######################################
# 填写 zookeeper集群列表
######################################
efak.zk.cluster.alias=cluster1
cluster1.zk.list={ZOOK_SERVERS}
######################################
# broker 最大规模数量
######################################
cluster1.efak.broker.size=20
######################################
# zk 客户端线程数
######################################
kafka.zk.limit.size=32
######################################
# EFAK webui 端口
######################################
efak.webui.port=8048
######################################
# kafka offset storage
######################################
cluster1.efak.offset.storage=kafka
######################################
# kafka jmx uri
######################################
cluster1.efak.jmx.uri=service:jmx:rmi:///jndi/rmi://%s/jmxrmi
######################################
# kafka metrics 指标,默认存储15天
######################################
efak.metrics.charts=true
efak.metrics.retain=15
######################################
# kafka sql topic records max
######################################
efak.sql.topic.records.max=5000
efak.sql.topic.preview.records.max=10
######################################
# delete kafka topic token
######################################
efak.topic.token=keadmin
######################################
# kafka sqlite 数据库地址(需要修改存储路径)
######################################
efak.driver=org.sqlite.JDBC
efak.url=jdbc:sqlite:{EFAK_DATA_DIR}/db/ke.db
efak.username=root
efak.password=www.kafka-eagle.org
5.3 entrypoint.sh
# 变量
EFAK_DIR=/efak
EFAK_CONF=/efak/conf/system-config.properties
# 2、替换配置文件内容,后期ZK集群的地址通过ENV传递
sed -i s@{EFAK_DATA_DIR}@${EFAK_DIR}@g ${EFAK_CONF}
sed -i s@{ZOOK_SERVERS}@${ZOOK_SERVERS}@g ${EFAK_CONF}
# 3、启动efka
${EFAK_DIR}/bin/ke.sh start
tail -f ${EFAK_DIR}/logs/ke_console.out
5.4 构建镜像并推送仓库
docker build -t harbor.oldxu.net/base/efak:3.0.1 .
docker push harbor.oldxu.net/base/efak:3.0.1
六、迁移efak到k8s
03-efak-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: efak
spec:
replicas: 1
selector:
matchLabels:
app: efak
template:
metadata:
labels:
app: efak
spec:
imagePullSecrets:
- name: harbor-login
containers:
- name: efak
image: harbor.oldxu.net/base/efak:3.0.4
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 8048
env:
- name: ZOOK_SERVERS
value: "zookeeper-0.zk-svc:2181,zookeeper-1.zk-svc:2181,zookeeper-2.zk-svc:2181"
[root@master01 kafka-efak-Project]# dig @10.96.0.10 zookeeper-2.zk-svc.default.svc.cluster.local +short
10.244.0.39
[root@master01 kafka-efak-Project]# kubectl exec -it kafka-0 -- cat /etc/resolv.conf
nameserver 10.96.0.10
search default.svc.cluster.local svc.cluster.local cluster.local
options ndots:5
[root@master01 kafka-efak-Project]#
[root@master01 kafka-efak-Project]# #value: "zookeeper-0.zk-svc:2181,zookeeper-1.zk-svc:2181,zookeeper-2.zk-svc:2181"
04-efak-service.yaml
apiVersion: v1
kind: Service
metadata:
name: efak-svc
spec:
selector:
app: efak
ports:
- port: 8048
targetPort: 8048
05-efak-ingress.yaml
#apiVersion: networking.k8s.io/v1
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: efak-ingress
spec:
ingressClassName: "nginx"
rules:
- host: "efak.oldxu.net"
http:
paths:
- path: /
pathType: Prefix
backend:
serviceName: efak-svc
servicePort: 8048
#service:
# name: efak-svc
# port:
# number: 8048
访问
JMX状态 (李架构说最好别开JMX,有风险。 参考log4j2被攻击,就是通过JMX攻击的)
zookeeper状态
END