【Kafka】安装Kafka
1. 安装Kafka
Kafka 官网https://kafka.apache.org/
上传文件至云服务器
解压Kafka
[root@VM-8-7-centos opt]# cd /opt
[root@VM-8-7-centos opt]# mkdir moudle
[root@VM-8-7-centos opt]# cd software/
[root@VM-8-7-centos software]# tar -zxvf kafka_2.12-3.0.0.tgz -C /opt/moudle/
修改名称
[root@VM-8-7-centos moudle]# mv kafka_2.12-3.0.0/ kafka
[root@VM-8-7-centos bin]# pwd
/opt/moudle/kafka/bin
修改配置文件
[root@VM-8-7-centos kafka]# cd config/
[root@VM-8-7-centos config]# ls
connect-console-sink.properties connect-file-source.properties consumer.properties server.properties
connect-console-source.properties connect-log4j.properties kraft tools-log4j.properties
connect-distributed.properties connect-mirror-maker.properties log4j.properties trogdor.conf
connect-file-sink.properties connect-standalone.properties producer.properties zookeeper.properties
[root@VM-8-7-centos config]# vim server.properties
[root@VM-8-7-centos config]# pwd
/opt/moudle/kafka/config
修改地方1：broker.id
修改地方2：日志
log.dirs=/opt/module/kafka/datas
修改地方3：Zookeeper
zookeeper.connect=101.42.49.137:2181,139.9.133.30:2181,114.116.254.124:2181/kafka
下图为课程中的截图
2. Kafka使用
2.0 集群分发脚本xsync(重要)
scp安全拷贝
scp可以实现服务器与服务器之间的数据拷贝。（from server1 to server2）
2.0.1 scp命令
scp -r $pdir/$fname $user@$host:$pdir/$fname
命令 递归 要拷贝的文件路径/名称 目的地用户@主机:目的地路径/名称
[root@VM-8-7-centos moudle]# scp -r kafka/ root@139.9.133.30:/opt/moudle
The authenticity of host '139.9.133.30 (139.9.133.30)' can't be established.
ECDSA key fingerprint is SHA256:CNtTiB8P/OCKir8Dx2nVGb461U8fuwHh9v3saUBMFKo.
ECDSA key fingerprint is MD5:6f:89:b0:b1:d8:55:9c:1d:82:1e:6e:54:ac:ca:ae:fb.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '139.9.133.30' (ECDSA) to the list of known hosts.
root@139.9.133.30's password:
javax.ws.rs-api-2.1.1.jar 100% 124KB 3.5MB/s 00:00
osgi-resource-locator-1.0.3.jar 100% 19KB 996.1KB/s 00:00
zstd-jni-1.5.0-2.jar 100% 6604KB 508.0KB/s 00:13
reflections-0.9.12.jar 100% 103KB 494.4KB/s 00:00
jetty-servlet-9.4.43.v20210629.jar 100% 142KB 547.7KB/s 00:00
jakarta.annotation-api-1.3.5.jar 100% 24KB 541.1KB/s 00:00
2.0.2 rsync远程同步工具
rsync主要用于备份和镜像。具有速度快、避免复制相同内容和支持符号链接的优点。
rsync和scp区别：用rsync做文件的复制要比scp的速度快，rsync只对差异文件做更新。scp是把所有文件都复制过去。
rsync -av $pdir/$fname $user@$host:$pdir/$fname
命令 选项参数 要拷贝的文件路径/名称 目的地用户@主机:目的地路径/名称
2.0.3 写一个集群分发脚本xsync (Shell 脚本)
#!/bin/bash
#1. 判断参数个数
if [ $# -lt 1 ]
then
echo Not Enough Arguement!
exit;
fi
#2. 遍历集群所有机器
for host in 139.9.133.30 14.116.254.124
do
echo ==================== $host ====================
#3. 遍历所有目录，挨个发送
for file in $@
do
#4. 判断文件是否存在
if [ -e $file ]
then
#5. 获取父目录
pdir=$(cd -P $(dirname $file); pwd)
#6. 获取当前文件的名称
fname=$(basename $file)
ssh $host "mkdir -p $pdir"
rsync -av $pdir/$fname $host:$pdir
else
echo $file does not exists!
fi
done
done
修改脚本 xsync 具有执行权限
[atguigu@hadoop102 bin]$ chmod +x xsync
2.1 Zookeeper集群安装
2.1.1 先在第一台服务器上安装
# 解压
[root@VM-8-7-centos software]# tar -zxvf apache-zookeeper-3.6.1-bin.tar.gz -C /opt/moudle/
[root@VM-8-7-centos moudle]# pwd
/opt/moudle
[root@VM-8-7-centos moudle]# ls
apache-zookeeper-3.6.1-bin kafka
# 改名
[root@VM-8-7-centos moudle]# mv apache-zookeeper-3.6.1-bin/ zookeeper-3.6.1
[root@VM-8-7-centos moudle]# ls
kafka zookeeper-3.6.1
[root@VM-8-7-centos zookeeper-3.6.1]# ll
total 40
drwxr-xr-x 2 lighthouse lighthouse 4096 Apr 21 2020 bin
drwxr-xr-x 2 lighthouse lighthouse 4096 Apr 21 2020 conf
drwxr-xr-x 5 lighthouse lighthouse 4096 Apr 21 2020 docs
drwxr-xr-x 2 root root 4096 Apr 21 17:01 lib
-rw-r--r-- 1 lighthouse lighthouse 11358 Apr 21 2020 LICENSE.txt
-rw-r--r-- 1 lighthouse lighthouse 432 Apr 21 2020 NOTICE.txt
-rw-r--r-- 1 lighthouse lighthouse 1963 Apr 21 2020 README.md
-rw-r--r-- 1 lighthouse lighthouse 3166 Apr 21 2020 README_packaging.md
[root@VM-8-7-centos zookeeper-3.6.1]# pwd
/opt/moudle/zookeeper-3.6.1
配置
[root@VM-8-7-centos zookeeper-3.6.1]# pwd
/opt/moudle/zookeeper-3.6.1
[root@VM-8-7-centos zookeeper-3.6.1]# ls
bin conf docs lib LICENSE.txt NOTICE.txt README.md README_packaging.md
[root@VM-8-7-centos zookeeper-3.6.1]# cd conf
[root@VM-8-7-centos conf]# ls
configuration.xsl log4j.properties zoo_sample.cfg
[root@VM-8-7-centos conf]# ll
total 12
-rw-r--r-- 1 lighthouse lighthouse 535 Apr 21 2020 configuration.xsl
-rw-r--r-- 1 lighthouse lighthouse 3435 Apr 21 2020 log4j.properties
-rw-r--r-- 1 lighthouse lighthouse 1148 Apr 21 2020 zoo_sample.cfg
[root@VM-8-7-centos conf]# ls
configuration.xsl log4j.properties zoo_sample.cfg
[root@VM-8-7-centos conf]# mv zoo_sample.cfg zoo.cfg
[root@VM-8-7-centos conf]# ls
configuration.xsl log4j.properties zoo.cfg
[root@VM-8-7-centos conf]# pwd
/opt/moudle/zookeeper-3.6.1/conf
# 修改快照数据存储目录
新建一个目录 /opt/moudle/zookeeper-3.6.1/zkData
修改zoo.cfg如下图
单机启动zookeeper
# 启动服务端
[root@VM-8-7-centos zookeeper-3.6.1]# /opt/moudle/zookeeper-3.6.1/bin/zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /opt/moudle/zookeeper-3.6.1/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@VM-8-7-centos zookeeper-3.6.1]# jps
1632 Jps
1550 QuorumPeerMain
[root@VM-8-7-centos zookeeper-3.6.1]# pwd
/opt/moudle/zookeeper-3.6.1
启动客户端
[root@VM-8-7-centos zookeeper-3.6.1]# /opt/moudle/zookeeper-3.6.1/bin/zkCli.sh
Connecting to localhost:2181
2024-04-21 17:39:01,844 [myid:] - INFO [main:Environment@98] - Client environment:zookeeper.version=3.6.1--104dcb3e3fb464b30c5186d229e00af9f332524b, built on 04/21/2020 15:01 GMT
2024-04-21 17:39:01,848 [myid:] - INFO [main:Environment@98] - Client environment:host.name=VM-8-7-centos
2024-04-21 17:39:01,848 [myid:] - INFO [main:Environment@98] - Client environment:java.version=1.8.0_131
2024-04-21 17:39:01,851 [myid:] - INFO [main:Environment@98] - Client environment:java.vendor=Oracle Corporation
2024-04-21 17:39:01,851 [myid:] - INFO [main:Environment@98] - Client environment:java.home=/usr/local/java/jdk1.8.0_131/jre
2024-04-21 17:39:01,851 [myid:] - INFO [main:Environment@98] - Client environment:java.class.path=/opt/moudle/zookeeper-3.6.1/bin/../zookeeper-server/target/classes:/opt/moudle/zookeeper-3.6.1/bin/../build/classes:/opt/moudle/zookeeper-3.6.1/bin/../zookeeper-server/target/lib/*.jar:/opt/moudle/zookeeper-3.6.1/bin/../build/lib/*.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/zookeeper-prometheus-metrics-3.6.1.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/zookeeper-jute-3.6.1.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/zookeeper-3.6.1.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/snappy-java-1.1.7.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/slf4j-log4j12-1.7.25.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/slf4j-api-1.7.25.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/simpleclient_servlet-0.6.0.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/simpleclient_hotspot-0.6.0.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/simpleclient_common-0.6.0.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/simpleclient-0.6.0.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/netty-transport-native-unix-common-4.1.48.Final.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/netty-transport-native-epoll-4.1.48.Final.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/netty-transport-4.1.48.Final.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/netty-resolver-4.1.48.Final.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/netty-handler-4.1.48.Final.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/netty-common-4.1.48.Final.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/netty-codec-4.1.48.Final.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/netty-buffer-4.1.48.Final.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/metrics-core-3.2.5.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/log4j-1.2.17.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/json-simple-1.1.1.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jline-2.11.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jetty-util-9.4.24.v20191120.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jetty-servlet-9.4.24.v20191120.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jetty-server-9.4.24.v20191120.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jetty-security-9.4.24.v20191120.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jetty-io-9.4.24.v20191120.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jetty-http-9.4.24.v20191120.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/javax.servlet-api-3.1.0.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jackson-databind-2.10.3.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jackson-core-2.10.3.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/jackson-annotations-2.10.3.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/commons-lang-2.6.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/commons-cli-1.2.jar:/opt/moudle/zookeeper-3.6.1/bin/../lib/audience-annotations-0.5.0.jar:/opt/moudle/zookeeper-3.6.1/bin/../zookeeper-*.jar:/opt/moudle/zookeeper-3.6.1/bin/../zookeeper-server/src/main/resources/lib/*.jar:/opt/moudle/zookeeper-3.6.1/bin/../conf:$:CLASSPATH:/usr/local/java/jdk1.8.0_131/lib/
2024-04-21 17:39:01,851 [myid:] - INFO [main:Environment@98] - Client environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
2024-04-21 17:39:01,851 [myid:] - INFO [main:Environment@98] - Client environment:java.io.tmpdir=/tmp
2024-04-21 17:39:01,851 [myid:] - INFO [main:Environment@98] - Client environment:java.compiler=<NA>
2024-04-21 17:39:01,851 [myid:] - INFO [main:Environment@98] - Client environment:os.name=Linux
2024-04-21 17:39:01,851 [myid:] - INFO [main:Environment@98] - Client environment:os.arch=amd64
2024-04-21 17:39:01,852 [myid:] - INFO [main:Environment@98] - Client environment:os.version=3.10.0-1160.71.1.el7.x86_64
2024-04-21 17:39:01,852 [myid:] - INFO [main:Environment@98] - Client environment:user.name=root
2024-04-21 17:39:01,852 [myid:] - INFO [main:Environment@98] - Client environment:user.home=/root
2024-04-21 17:39:01,852 [myid:] - INFO [main:Environment@98] - Client environment:user.dir=/opt/moudle/zookeeper-3.6.1
2024-04-21 17:39:01,852 [myid:] - INFO [main:Environment@98] - Client environment:os.memory.free=25MB
2024-04-21 17:39:01,854 [myid:] - INFO [main:Environment@98] - Client environment:os.memory.max=228MB
2024-04-21 17:39:01,854 [myid:] - INFO [main:Environment@98] - Client environment:os.memory.total=31MB
2024-04-21 17:39:01,858 [myid:] - INFO [main:ZooKeeper@1005] - Initiating client connection, connectString=localhost:2181 sessionTimeout=30000 watcher=org.apache.zookeeper.ZooKeeperMain$MyWatcher@4926097b
2024-04-21 17:39:01,863 [myid:] - INFO [main:X509Util@77] - Setting -D jdk.tls.rejectClientInitiatedRenegotiation=true to disable client-initiated TLS renegotiation
2024-04-21 17:39:01,870 [myid:] - INFO [main:ClientCnxnSocket@239] - jute.maxbuffer value is 1048575 Bytes
2024-04-21 17:39:01,877 [myid:] - INFO [main:ClientCnxn@1703] - zookeeper.request.timeout value is 0. feature enabled=false
2024-04-21 17:39:01,883 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1154] - Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181.
2024-04-21 17:39:01,883 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1156] - SASL config status: Will not attempt to authenticate using SASL (unknown error)
Welcome to ZooKeeper!
JLine support is enabled
2024-04-21 17:39:01,990 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@986] - Socket connection established, initiating session, client: /0:0:0:0:0:0:0:1:39710, server: localhost/0:0:0:0:0:0:0:1:2181
2024-04-21 17:39:02,026 [myid:localhost:2181] - INFO [main-SendThread(localhost:2181):ClientCnxn$SendThread@1420] - Session establishment complete on server localhost/0:0:0:0:0:0:0:1:2181, session id = 0x107b37b376f0000, negotiated timeout = 30000
WATCHER::
WatchedEvent state:SyncConnected type:None path:null
查看状态
[zk: localhost:2181(CONNECTED) 1] [root@VM-8-7-centos zookeeper-3.6.1]#
[root@VM-8-7-centos zookeeper-3.6.1]# bin/zkServer.sh status
或者 [root@VM-8-7-centos zookeeper-3.6.1]# /opt/moudle/zookeeper-3.6.1/bin/zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /opt/moudle/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: standalone # 这个指单机
[root@VM-8-7-centos zookeeper-3.6.1]#
停服务
[root@VM-8-7-centos zookeeper-3.6.1]# bin/zkServer.sh stop
ZooKeeper JMX enabled by default
Using config: /opt/moudle/zookeeper-3.6.1/bin/../conf/zoo.cfg
Stopping zookeeper ... STOPPED
Zookeeper的一些配置参数
2.1.2 在三台服务器上都安装Zookeeper
先在第一台服务器上，补充创建myid
分发数据
[root@VM-8-7-centos moudle]# xsync zookeeper-3.6.1/
修改myid 两台服务器一个为1 另一个为2
增加配置，三台服务器都要加
[root@VM-8-7-centos zookeeper-3.6.1]# cd conf
[root@VM-8-7-centos conf]# ls
configuration.xsl log4j.properties zoo.cfg
[root@VM-8-7-centos conf]# vim zoo.cfg
server.0=101.42.49.137:2888:3888
server.1=139.9.133.30:2888:3888
server.2=114.116.254.124:2888:3888
quorumListenOnAllIPs=true
[root@VM-8-7-centos conf]# pwd
/opt/moudle/zookeeper-3.6.1/conf
补充：需要对所有的服务器防火墙打开端口的限制
2.2 对Kafka进行分发
2.2.1 执行同步脚本
2.2.2 三台云主机配置Kafka环境变量
# jdk1.8 (有两台不一样，zookeeper需要jdk环境)
export JAVA_HOME=/usr/local/java/jdk1.8.0_131
export CLASSPATH=$:CLASSPATH:$JAVA_HOME/lib/
export PATH=$PATH:$JAVA_HOME/bin
# Nodejs
export PATH=/usr/local/node/node-v12.16.3-linux-x64/bin:$PATH
# maven
export MAVEN_HOME=/opt/maven/apache-maven-3.6.3
export PATH=$MAVEN_HOME/bin:$PATH
# kafka
export KAFKA_HOME=/opt/module/kafka
export PATH=$PATH:$KAFKA_HOME/bin
[root@VM-8-7-centos moudle]# source /etc/profile
2.3 启动kafka集群
[root@VM-8-7-centos moudle]# cd kafka/
[root@VM-8-7-centos kafka]# ll
total 64
drwxr-xr-x 3 root root 4096 Sep 9 2021 bin
drwxr-xr-x 3 root root 4096 Apr 21 16:25 config
drwxr-xr-x 2 root root 4096 Apr 6 17:20 libs
-rw-r--r-- 1 root root 14521 Sep 9 2021 LICENSE
drwxr-xr-x 2 root root 4096 Sep 9 2021 licenses
-rw-r--r-- 1 root root 28184 Sep 9 2021 NOTICE
drwxr-xr-x 2 root root 4096 Sep 9 2021 site-docs
[root@VM-8-7-centos kafka]# /opt/moudle/kafka/bin/kafka-server-start.sh -daemon config/server.properties
[root@VM-8-7-centos kafka]# jps
24662 Jps
21086 QuorumPeerMain
24511 Kafka
[root@VM-8-7-centos kafka]# pwd
/opt/moudle/kafka
在其他两台服务器也这么启动
2.3.1 kafka启动停止脚本
[root@VM-8-7-centos bin]# pwd
/bin
[root@VM-8-7-centos bin]# vim kf.sh
[root@VM-8-7-centos bin]# chmod +x kf.sh
启动集群命令 kf.sh start
停止集群命令 kf.sh stop
#! /bin/bash
case $1 in
"start"){
for i in 101.42.49.137 139.9.133.30 114.116.254.124
do
echo " --------启动 $i Kafka-------"
ssh $i "/opt/moudle/kafka/bin/kafka-server-start.sh -daemon /opt/moudle/kafka/config/server.properties"
done
};;
"stop"){
for i in 101.42.49.137 139.9.133.30 114.116.254.124
do
echo " --------停止 $i Kafka-------"
ssh $i "/opt/moudle/kafka/bin/kafka-server-stop.sh "
done
};;
esac
3.项目中的使用
3.1 yml中的配置
kafka:
preStaff:
topicName: onboard-.........
ip: 10.134.xx.233:9092,xxx:9092
groupId: onboard_xxx
username: consumer
password: consumer1-1secret
er:
topicName: personnel-budget-local
ip: 10.134.xx.233:9092,xxx:9092
groupId: personnel_xxx
username: consumer
password: consumer1-1secret
producer:
topicName:
hcChange: hc_change_to_recruitxxxxxx
hcNormalCalculate: hr-budget-normal-calculatxxxx
hcActivityCalculate: hr-budget-activity-calculxx
hcForecastCalculate: hr-budget-forecast-calculate-xxx
hcCalculateUtilCalculate: hr-budget-calculate-util-txxx
hcCalculateUtilTemp: hr-budget-calculate-xxx-xxx-xxx-local
hcActivityDeptCalculate: hr-budget-activity-xxx-xxx-topic-local
calculateUtilDeptTopicName: hr-budget-calculate-util-xxx-topic-local
消费者：
/**
* 消费预入职kafka
*
* @param record 消息
* @param ack ack
*/
@KafkaListener(topics = {"#{'${kafka.preStaff.topicName}'}"}, containerFactory = "PreStaffKafkaFactory")
public void preStaffConsumer(ConsumerRecord<?, ?> record, Acknowledgment ack) {
Optional<?> kafkaMessage = Optional.ofNullable(record.value());
if (kafkaMessage.isPresent()) {
// 业务逻辑
} catch (Exception e) {
log.error(...);
} finally {
ack.acknowledge();
}
}
生产者：
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.kafka.support.SendResult;
import org.springframework.stereotype.Component;
import org.springframework.util.concurrent.ListenableFuture;
import org.springframework.util.concurrent.ListenableFutureCallback;
import java.util.concurrent.ExecutionException;
@Component
@Slf4j
public class KafkaProducer {
@Autowired
private KafkaTemplate<String, String> kafkaTemplate;
public void send(String topicName, Object data) throws ExecutionException, InterruptedException {
String kafkaMessage = JSONObject.toJSONString(data);
log.info("kafka准备发送消息: topic:{} message:{}", topicName, kafkaMessage);
ListenableFuture<SendResult<String, String>> future = kafkaTemplate.send(topicName, kafkaMessage);
future.addCallback(new ListenableFutureCallback<SendResult<String, String>>() {
@Override
public void onFailure(Throwable throwable) {
//失败回调
log.error("OSD-BUDGET生产者发送消息失败: topic:{}, SendResult:{}", topicName, throwable.getMessage());
}
@Override
public void onSuccess(SendResult<String, String> stringObjectSendResult) {
//成功回调
log.info("OSD-BUDGET生产者发送消息成功: topic:{}, SendResult:{}", topicName, stringObjectSendResult.toString());
}
});
kafkaTemplate.flush();
}
}
1532
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
