kafka集群+knowstreaming安装

环境：centos7 mysql5.7 :kafka_2.13-3.5.1.tgz KnowStreaming-3.0.0-beta.3.tar.gz jdk11.tar.gz elasticsearch:7.6.1
三台centos7机器分别为：192.168.50.154 192.168.50.155 192.168.50.156
1.安装mysql脚本（作为参考）

function Install_Mysql(){
        cd $dir
        rpm -qa | grep -E "mariadb|mysql" | xargs yum -y remove >/dev/null 2>&1
        mv -f /var/lib/mysql/ /var/lib/mysqlbak$(date "+%s") >/dev/null 2>&1
        mkdir -p $dir/mysql/ && cd $dir/mysql/
        tar -zxf $dir/mysql5.7.tar.gz -C $dir/mysql/
        yum -y localinstall mysql* libaio*
        systemctl start mysqld
        systemctl enable mysqld >/dev/null 2>&1
        old_pass=`grep 'temporary password' /var/log/mysqld.log | awk '{print $NF}' | tail -n 1`
        mysql -NBe "alter user USER() identified by '$mysql_pass';" --connect-expired-password -uroot -p$old_pass
        if [ $? -eq 0 ];then
                echo  "Mysql database installation completed"
        else
                echo "Mysql database configuration failed. The script exits"
                exit
        fi
}

dir=`pwd`
mysql_pass=`date +%s |sha256sum |base64 |head -c 10 ;echo`"_Di2"
echo "$mysql_pass" > $dir/mysql.password
Install_Mysql

2.安装java11

function Install_Java(){
        cd $dir
    tar -zxf $dir/jdk11.tar.gz -C /usr/local/
    mv -f /usr/local/jdk-11.0.2 /usr/local/java11 >/dev/null 2>&1
    echo "export JAVA_HOME=/usr/local/java11" >> ~/.bashrc
    echo "export CLASSPATH=/usr/java/java11/lib" >> ~/.bashrc
    echo "export PATH=\$JAVA_HOME/bin:\$PATH:\$HOME/bin" >> ~/.bashrc
    source ~/.bashrc
}
dir=`pwd`
Install_Java

3.安装elatiscearch

function Install_ElasticSearch(){
        kill -9 $(ps -ef | grep elasticsearch | grep -v "grep" | awk '{print $2}')  >/dev/null 2>&1
        id esuser  >/dev/null 2>&1
        if [ "$?" != "0" ];then
                useradd esuser
                echo "esuser soft nofile 655350" >>/etc/security/limits.conf
                echo "esuser hard nofile 655350" >>/etc/security/limits.conf
                echo "vm.max_map_count = 655360" >>/etc/sysctl.conf
                sysctl -p >/dev/null 2>&1
        fi
        mkdir -p /km_es/es_data  && cd /km_es/ >/dev/null 2>&1
        tar -zxf $dir/elasticsearch-7.6.1.tar.gz -C /km_es/
        chown -R esuser:esuser /km_es/
        su - esuser <<-EOF
                export JAVA_HOME=/usr/local/java11
                sh /km_es/elasticsearch-7.6.1/control.sh start
        EOF
        sleep 5
        es_status=`sh /km_es/elasticsearch-7.6.1/control.sh status | grep -o "started"`
        if [ "$es_status" = "started" ];then
                echo "elasticsearch started successfully～ "
        else
                echo "Elasticsearch failed to start. The script exited"
                exit
        fi
}

4.安装KnowStreaming

function Install_KnowStreaming(){
        cd $dir
        tar -zxf KnowStreaming-3.0.0-beta.3.tar.gz -C $dir/
        mysql -uroot -p$mysql_pass -e "create database know_streaming;"
        mysql -uroot -p$mysql_pass  know_streaming < ./KnowStreaming/init/sql/ddl-ks-km.sql
        mysql -uroot -p$mysql_pass  know_streaming < ./KnowStreaming/init/sql/ddl-logi-job.sql
    mysql -uroot -p$mysql_pass  know_streaming < ./KnowStreaming/init/sql/ddl-logi-security.sql
    mysql -uroot -p$mysql_pass  know_streaming < ./KnowStreaming/init/sql/dml-ks-km.sql
    mysql -uroot -p$mysql_pass  know_streaming < ./KnowStreaming/init/sql/dml-logi.sql
    sleep 10
    sh ./KnowStreaming/init/template/template.sh
    sed -i "s/mysql_pass/"$mysql_pass"/g" ./KnowStreaming/conf/application.yml
    sed -i "s/address:.*/address: 127.0.0.1:8060/g" ./KnowStreaming/conf/application.yml
    cd $dir/KnowStreaming/bin/ && sh startup.sh

}

5.安装kafka

tar -xvf kafka_2.13-3.5.1.tgz
mv kafka_2.13-3.5.1.tgz kafka


## server.properties配置  192.168.50.154
## 三台机器的配置broker.id（分别为1 2 3 ）  listeners各不一样
broker.id=1
listeners=PLAINTEXT://192.168.50.154:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka
num.partitions=3
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=4
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.50.154:2181,192.168.50.155:2181,192.168.50.156:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
auto.create.topics.enable=true
delete.topic.enable=true



## zookeeper.properties配置
dataDir=/data/zookeeper/data
clientPort=2181
maxClientCnxns=0
admin.enableServer=false
server.1=192.168.50.154:2888:3888
server.2=192.168.50.155:2888:3888
server.3=192.168.50.156:2888:3888
initLimit=10
syncLimit=5
tickTime=2000



####
mkdir -p /data/zookeeper/data  #创建数据目录
cat /data/zookeeper/data/myid
###配置zookeeper-server-start.sh
#添加java路径
vim /app/kafka/bin/zookeeper-server-start.sh
export JAVA_HOME=/usr/local/java11
export PATH=$JAVA_HOME/bin:$PATH


###配置kafka-server-start.sh
export JAVA_HOME=/usr/local/java11
export PATH=$JAVA_HOME/bin:$PATH
export JMX_PORT=9999




# 做成服务
[root@first-1 system]# cat zookeeper.service 
[Unit]
Description=zookeeper server
Requires=network.target
After=network.target

[Service]
Type=simple
ExecStart=/app/kafka/bin/zookeeper-server-start.sh /app/kafka/config/zookeeper.properties
ExecStop=/app/kafka/bin/zookeeper-server-stop.sh
Restart=on-failure
User=root
Group=root

[Install]
WantedBy=multi-user.target


[root@first-1 system]# cat kafka.service 
[Unit]
Description=kafka server
Requires=network.target
After=network.target

[Service]
Type=simple
ExecStart=/app/kafka/bin/kafka-server-start.sh /app/kafka/config/server.properties
ExecStop=/app/kafka/bin/kafka-server-stop.sh
Restart=on-failure
User=root
Group=root

[Install]
WantedBy=multi-user.target


# 分别使用systemctl启动zookeeper和kafka，注意点：先启动zookeeper

5 安装的过程需要注意的点
a 可能出现无法使用root用户登录。必须使用esuser登录
b template目录下的template.sh中脚本有错

set -x
esaddr=192.168.50.154
port=8060
#curl -s --connect-timeout 10 -o /dev/null  http://${esaddr}:${port}/_cat/nodes >/dev/null 2>&1 
#if [ "$?" != "0" ];then
#    echo "Elasticserach 访问失败, 请安装完后检查并重新执行该脚本 "
#    exit
#fi
wget -q --spider --timeout=10 http://192.168.50.154:8060/
if [ "$?" != "0" ];then
    echo "Elasticserach 访问失败, 请安装完后检查并重新执行该脚本 "
    exit
fi

c 使用命令行新建一个topic测试。再访问8080端口检查是否有对应的topic

./kafka-topics.sh --create --bootstrap-server 192.168.50.154:9092 --replication-factor 3 --partitions 1 --topic topic1

d 可能需要注意的点，有的时候会发现无法kafka集群无法连接。也不用过于在意。查看本地端口3888起来了，并且防火墙之类的都关起来了，就不用太担心，使用systemclt启动要好一点。多看日志
e 参考官方文档：
脚本部署
手动安装
f 展示