二、kafka集群搭建

1、制作Kafka镜像

1.1 Dockerfile

FROM harbor.xingxingcar.com/library/jdk:8-centos7

#1、调整时区
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && \
    echo "Asia/Shanghai" > /etc/timezone

#2、拷贝软件及配置
ENV VERSION=2.12-2.2.0
ADD ./kafka_${VERSION}.tgz /
ADD ./server.properties /kafka_${VERSION}/config/server.properties

#3、重命名目录
RUN mv /kafka_${VERSION} /kafka

#4、启动脚本
ADD entrypoint.sh /entrypoint.sh

#5、暴露kafka端口 
EXPOSE 9092 9999

#6、启动脚本
CMD ["/bin/bash","/entrypoint.sh"]

1.2 server.properties

############################# Server Basics ############################# 
# broker的id，值为整数，且必须唯一，在一个集群中不能重复
broker.id={BROKER_ID}

############################# Socket Server Settings ############################# 
# kafka监听端口，默认9092
listeners=PLAINTEXT://{LISTENERS}:9092

# 处理网络请求的线程数量，默认为3个
num.network.threads=3

# 执行磁盘IO操作的线程数量，默认为8个 
num.io.threads=8

# socket服务发送数据的缓冲区大小，默认100KB
socket.send.buffer.bytes=102400

# socket服务接受数据的缓冲区大小，默认100KB
socket.receive.buffer.bytes=102400

# socket服务所能接受的一个请求的最大大小，默认为100M
socket.request.max.bytes=104857600

############################# Log Basics ############################# 
# kafka存储消息数据的目录
log.dirs={KAFKA_DATA_DIR}

# 每个topic默认的partition
num.partitions=1

# 设置副本数量为3,当Leader的Replication故障，会进行故障自动转移。
default.replication.factor=3

# 在启动时恢复数据和关闭时刷新数据时每个数据目录的线程数量
num.recovery.threads.per.data.dir=1

############################# Log Flush Policy ############################# 
# 消息刷新到磁盘中的消息条数阈值
log.flush.interval.messages=10000

# 消息刷新到磁盘中的最大时间间隔,1s
log.flush.interval.ms=1000

############################# Log Retention Policy ############################# 
# 日志保留小时数，超时会自动删除，默认为7天
log.retention.hours=168

# 日志保留大小，超出大小会自动删除，默认为1G
#log.retention.bytes=1073741824

# 日志分片策略，单个日志文件的大小最大为1G，超出后则创建一个新的日志文件
log.segment.bytes=1073741824

# 每隔多长时间检测数据是否达到删除条件,300s
log.retention.check.interval.ms=300000

############################# Zookeeper ############################# 
# Zookeeper连接信息，如果是zookeeper集群，则以逗号隔开
zookeeper.connect={ZK_SERVERS}

# 连接zookeeper的超时时间,6s
zookeeper.connection.timeout.ms=6000

1.3 entrypoint.sh

#变量
KAFKA_DIR=/kafka
KAFKA_CONF=/kafka/config/server.properties

#1、基于主机名+1 获取Broker_id 这个是用来标识集群节点在整个集群中必须唯一
BROKER_ID=$(( $(hostname | sed 's#.*-##g') + 1 ))
LISTENERS=$(hostname -i)

#2、替换配置文件内容，ZK地址后面传递变量替换
sed -i s@{BROKER_ID}@${BROKER_ID}@g ${KAFKA_CONF}
sed -i s@{LISTENERS}@${LISTENERS}@g ${KAFKA_CONF}
sed -i s@{KAFKA_DATA_DIR}@${KAFKA_DATA_DIR-/data}@g ${KAFKA_CONF}
sed -i s@{ZK_SERVERS}@${ZK_SERVERS}@g ${KAFKA_CONF}

#3、启动Kafka
cd ${KAFKA_DIR}/bin
sed -i '/export KAFKA_HEAP_OPTS/a export JMX_PORT="9999"' kafka-server-start.sh
./kafka-server-start.sh ../config/server.properties

2. 编写yaml文件

[admin@master01 kafka]$cat yaml/kafka.yaml 
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: kafka
  namespace: logging
spec:
  serviceName: kafka-svc
  replicas: 3
  selector:
    matchLabels:
      app: kafka
  template:
    metadata:
      labels:
        app: kafka
    spec:
      imagePullSecrets:
      - name: harbor-admin
      containers:
      - name: kafka
        image: harbor.xx.com/library/kafka:2.12 
        imagePullPolicy: Always       
        ports:
        - name: client 
          containerPort: 9092
        - name: jmxport
          containerPort: 9999
        env: 
        - name: ZK_SERVERS 
          value: "zookeeper-0.zk-svc:2181,zookeeper-1.zk-svc:2181,zookeeper-2.zk-svc:2181"
        volumeMounts:
        - name: data
          mountPath: /data


  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteMany"]
      storageClassName: "nfs-provisioner-storage"
      resources:
        requests:  
          storage: 50Gi

---
apiVersion: v1
kind: Service
metadata:
  name: kafka-svc
  namespace: logging
spec:
  clusterIP: None
  ports: 
  - name: client
    port: 9092
    targetPort: 9092
  - name: jmx
    port: 9999
    targetPort: 9999  

3. 检查集群

3.1 查看pod ，service

[admin@master01 ~]$kubectl  get pod,service -n logging
NAME              READY   STATUS    RESTARTS   AGE
pod/kafka-0       1/1     Running   0          15h
pod/kafka-1       1/1     Running   0          15h
pod/kafka-2       1/1     Running   0          15h
pod/zookeeper-0   1/1     Running   0          19h
pod/zookeeper-1   1/1     Running   0          19h
pod/zookeeper-2   1/1     Running   0          19h

NAME                TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                      AGE
service/kafka-svc   ClusterIP   None         <none>        9092/TCP,9999/TCP            15h
service/zk-svc      ClusterIP   None         <none>        2181/TCP,2888/TCP,3888/TCP   19h

3.2 使用kafka命令创建一个topic

root@kafka-0:/# /kafka/bin/kafka-topics.sh \
> --create \
> --zookeeper zookeeper-0.zk-svc:2181,zookeeper-1.zk-svc:2181,zookeeper-2.zk-svc:2181 \
> --partitions 1 \
> --replication-factor 3 \
> --topic ikun
Created topic ikun.

3.3 模拟消息发布者

root@kafka-0:/# /kafka/bin/kafka-console-producer.sh --broker-list kafka-0.kafka-svc:9092,kafka-1.kafka-svc:9092,kafka-2.kafka-svc:9092 --topic ikun
>hello ikun
>hello kunkun

3.4 模拟消息订阅者

root@kafka-2:/# /kafka/bin/kafka-console-consumer.sh --bootstrap-server  kafka-0.kafka-svc:9092,kafka-1.kafka-svc:9092,kafka-2.kafka-svc:9092 --topic ikun --from-beginning 

hello ikun
hello kunkun