批量部署【KAFKA集群3~n】-通过【ansible】剧本机制

安装kafka_2.12-3.6.1.tgz 集群3节点实例，非kraft模式

制品信息介绍

kafka 安装包，从官网下载：

• kafka_2.12-3.6.1.tgz
• apache-zookeeper-3.8.3.tar.gz

安装步骤

命令行批量删除命令

# ansible k8s -m file -a "path=/mnt/kafka_2.13-3.6.1.tgz state=absent"

安装ansible

自行百度

安装zookeeper

• 安装zookeeper：复制本机软件包到目标机器，包含Zookeeper、Kafka的复制
• 配置zookeeper.yaml: 配置信息如下

## 安装zookeeper
---
- hosts: k8s
  remote_user: root
  tasks:
    - name: copy apache-zookeeper-3.8.3
      copy:
        src: "{{ item }}"
        dest: /mnt
      with_fileglob:
        - ./apache-zookeeper-3.8.3-bin.tar.gz
        - ./zoo.cfg
      tags:
        - copyPackage
    - name: create zk data log if it does not exist
      file:
        path: "{{ item }}"
        state: directory
      loop:
        - /data1/zookeeper/data
        - /data1/zookeeper/log
    - name: unarchive apache-zookeeper-3.8.3-bin.tar.gz
      unarchive:
        src: /mnt/apache-zookeeper-3.8.3-bin.tar.gz
        dest: /mnt/
    - name: Rename directory after extraction
      file:
        dest: /mnt/apache-zookeeper-3.8.3
        src:  /mnt/apache-zookeeper-3.8.3-bin
        state: link
      tags:
        - link
    - name: install apache-zookeeper-3.8.3
      shell: cp ./apache-zookeeper-3.8.3/conf/zoo_sample.cfg ./apache-zookeeper-3.8.3/conf/zoo.cfg;
      args:
        chdir: /mnt
      tags:
        - installApache
    - name: udpate zoo.cfg
      shell: cat ./zoo.cfg > ./apache-zookeeper-3.8.3/conf/zoo.cfg
      args:
        chdir: /mnt
      tags:
        - UpdateZoocfg


# 每个节点执行自己的标识，通常+1
- hosts: "{{ groups['k8s'][0] }}"
  remote_user: root
  tasks:
    - name: mkdir zk data if it does not exist
      file:
        path: /data1/zookeeper/data
        state: directory
    - name: config zookeeper mysid1
      shell: echo 1 > /data1/zookeeper/data/myid;
      tags:
        - configMysid

- hosts: "{{ groups['k8s'][1] }}"
  remote_user: root
  tasks:
    - name: mkdir zk data if it does not exist
      file:
        path: /data1/zookeeper/data
        state: directory
    - name: config zookeeper mysid2
      shell: echo 2 > /data1/zookeeper/data/myid;
      tags:
        - configMysid

- hosts: "{{ groups['k8s'][2] }}"
  remote_user: root
  tasks:
    - name: mkdir zk data if it does not exist
      file:
        path: /data1/zookeeper/data
        state: directory
    - name: config zookeeper mysid3
      shell: echo 3 > /data1/zookeeper/data/myid;
      tags:
        - configMysid

# 启动zookeeper集群,tags标识startZK
- hosts: k8s
  remote_user: root
  tasks:
    - name: start zookeeper
      shell: ./apache-zookeeper-3.8.3/bin/zkServer.sh start
      args:
        chdir: /mnt
      tags:
        - startZk
# 重新启动zookeeper集群,tags标识restartZK
- hosts: k8s
  remote_user: root
  tasks:
    - name: restart zookeeper
      shell: ./apache-zookeeper-3.8.3/bin/zkServer.sh restart
      args:
        chdir: /mnt
      tags:
        - restartZk

# ansible-playbook  zookeeper.yaml --check  
# ansible-playbook  zookeeper.yaml
查看当前节点ZK状态
# sh zkServer.sh status

运行结果

安装kafka

ansible-playbook kafka.yaml

kafka剧本信息

## 安装kafka
---
- hosts: k8s
  remote_user: root
  tasks:
    - name: copy kafka_2.13-3.6.1.tgz
      copy:
        src: "{{ item }}"
        dest: /mnt
      with_fileglob:
        - ./kafka_2.13-3.6.1.tgz
        - ./server.properties
      tags:
        - copyPackage
    - name: create kafka data file
      file:
        path: /data1/kafka
        state: directory
    - name: unarchive kafka_2.13-3.6.1
      unarchive:
        src: /mnt/kafka_2.13-3.6.1.tgz
        dest: /mnt/
      tags:
        - unarchive
    - name: update server.properties
      shell: cat server.properties > /mnt/kafka_2.13-3.6.1/config/server.properties
      args:
        chdir: /mnt
      tags:
        - updateServerProperties

# 每个节点执行自己的标识，通常+1
- hosts: "{{ groups['k8s'][0] }}"
  remote_user: root
  tasks:
    - name: config zookeeper mysid1
      blockinfile:
        path: /mnt/kafka_2.13-3.6.1/config/server.properties
        block: |
          broker.id=1
          listeners=PLAINTEXT://{{ groups['k8s'][0] }}:9092
        marker: "# {mark} ANSIBLE MANAGED BLOCK FOR ZOO_CFG"
      tags:
        - config
        - configMysid1

- hosts: "{{ groups['k8s'][1] }}"
  remote_user: root
  tasks:
    - name: config zookeeper mysid2
      blockinfile:
        path: /mnt/kafka_2.13-3.6.1/config/server.properties
        block: |
          broker.id=2
          listeners=PLAINTEXT://{{ groups['k8s'][1] }}:9092
        marker: "# {mark} ANSIBLE MANAGED BLOCK FOR ZOO_CFG"
      tags:
        - config
        - configMysid2

- hosts: "{{ groups['k8s'][2] }}"
  remote_user: root
  tasks:
    - name: config zookeeper mysid3
      blockinfile:
        path: /mnt/kafka_2.13-3.6.1/config/server.properties
        block: |
          broker.id=3
          listeners=PLAINTEXT://{{ groups['k8s'][2] }}:9092
        marker: "# {mark} ANSIBLE MANAGED BLOCK FOR ZOO_CFG"
      tags:
        - config
        - configMysid3

## 启动start kafka
- hosts: k8s
  remote_user: root
  tasks:
    - name: start kafka
      shell: nohup ./kafka_2.13-3.6.1/bin/kafka-server-start.sh ./kafka_2.13-3.6.1/config/server.properties  > kafka.out 2>&1 &
      tags:
        - startKafka
      args:
        chdir: /mnt

运行结果：

zookeeper 配置

tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data1/zookeeper/data
dataLogDir=/data1/zookeeper/log
clientPort=2181
server.1=XXX:2888:3888
server.2=XXX:2888:3888
server.3=XXX:2888:3888

kafka 配置

server.properties配置信息

num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data1/kafka/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=XXX:2181,XXX:2181,XXX.cn:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0