该篇文章主要记录测试环境pulsar集群部署的流程,官网指路:Welcome to the Apache Pulsar documentation portal | Apache Pulsar*https://pulsar.apache.org/docs/3.1.x/
1.下载安装包并解压
tar -zvxf apache-pulsar-2.9.0-bin.tar.gz -C /opt/module/
重命名:
mv apache-pulsar-2.9.0 pulsar
2. 部署zookeeper
2.1 修改conf/zookeeper.conf
dataDir=/opt/module/pulsar/data1/zookeeper/data
dataLogDir=/opt/module/pulsar/data1/zookeeper/log
server.1=bigdata102:2888:3888
server.2=bigdata103:2888:3888
server.3=bigdata104:2888:3888
2.2 每台服务器上都创建目录
在节点1上:
mkdir -pv /opt/module/pulsar/data1/zookeeper/data
mkdir -pv /opt/module/pulsar/data1/zookeeper/ log
echo 1 > /opt/module/pulsar/data1/zookeeper/data/myid
分发 pulsar 目录到节点2,节点3
#集群分发脚本 xsync
xsync pulsar
依次修改 myid为2,3
2.3 后台启动zookeeper(3台同时)
bin/pulsar-daemon start zookeeper
2.4 检查zookeeper是否启动成功
ps -aux | grep zookeeper
2.5 元数据初始化
bin/pulsar initialize-cluster-metadata \
--cluster pulsar-cluster-zk \
--zookeeper bigdata102:2181 \
--configuration-store bigdata102:2181 \
--web-service-url http://bigdata102:8080 \
--web-service-url-tls https://bigdata102:8443 \
--broker-service-url pulsar://bigdata102:6650 \
--broker-service-url-tls pulsar+ssl://bigdata102:6651
2.6 查看初始化结果
bin/pulsar zookeeper-shell
3.部署BookKeeper集群
3.1 配置修改
advertisedAddress=bigdata102
zkServers=bigdata102:2181,bigdata103:2181,bigdata104:2181
journalDirectory=/opt/module/pulsar/data1/bookkeeper/journal
ledgerDirectories=/opt/module/pulsar/data1/bookkeeper/ledgers
prometheusStatsHttpPort=8100
3.2 每台服务器上都创建目录
mkdir -pv /opt/module/pulsar/data1/bookkeeper/journal
mkdir -pv /opt/module/pulsar/data1/bookkeeper/ledgers
3.3 执行初始化元数据命令
若出现提示,输入Y继续(该步骤只需在一个bookie节点执行一次,总共只需执行一次)
bin/bookkeeper shell metaformat
3.4 后台启动bookie(3台同时)
bin/pulsar-daemon start bookie
3.5验证是否启动成功
bin/bookkeeper shell bookiesanity
出现Bookie sanity test succeeded则代表启动成功。
4.部署Broker集群
4.1 配置修改
zookeeperServers=bigdata102:2181,bigdata103:2181,bigdata104:2181
configurationStoreServers=bigdata102:2181,bigdata103:2181,bigdata104:2181
advertisedAddress=bigdata102
# clusterName与前面zookeeper初始化的cluster一致
clusterName=pulsar-cluster-zk
4.2 后台启动broker(3台同时)
bin/pulsar-daemon start broker
4.3 查看集群 brokers 节点情况
bin/pulsar-admin brokers list pulsar-cluster
5. 验证集群
5.1查看集群节点
bin/pulsar-admin brokers list cluster-1
5.2 消费测试消息
bin/pulsar-client consume persistent://public/default/test -n 100 -s "consumer-test" -t "Exclusive"
5.3 发送测试消息
bin/pulsar-client produce persistent://public/default/test -n 1 -m "Hello Pulsar"
6.集群启停脚本
6.1集群分发脚本 xsync
在/root/bin 路径下 vim xsync
#!/bin/bash
#1. 判断参数个数
if [ $# -lt 1 ]
then
echo Not Enough Arguement!
exit;
fi
#2. 遍历集群所有机器
for host in hadoop01 hadoop02 hadoop03 hadoop04 hadoop05 hadoop06
do
echo ==================== $host ====================
#3. 遍历所有目录,挨个发送
for file in $@
do
#4. 判断文件是否存在
if [ -e $file ]
then
#5. 获取父目录
pdir=$(cd -P $(dirname $file); pwd)
#6. 获取当前文件的名称
fname=$(basename $file)
ssh $host "mkdir -p $pdir"
rsync -av $pdir/$fname $host:$pdir
else
echo $file does not exists!
fi
done
done
添加执行权限 chmod 777 xsync
6.2 ps.sh (pulsar集群的启停脚本)
在/root/bin 路径下 vim ps.sh
! /bin/bash
case $1 in
"start"){
for i in hadoop01 hadoop02 hadoop03 hadoop04 hadoop05 hadoop06
do
echo " --------启动 $i zookeeper-------"
ssh $i "/opt/module/pulsar/bin/pulsar-daemon start zookeeper"
echo " --------启动 $i bookKeeper-------"
ssh $i "/opt/module/pulsar/bin/pulsar-daemon start bookie"
echo " --------启动 $i broker-------"
ssh $i "/opt/module/pulsar/bin/pulsar-daemon start broker"
done
};;
"stop"){
for i in hadoop01 hadoop02 hadoop03 hadoop04 hadoop05 hadoop06
do
echo " --------停止 $i broker-------"
ssh $i "/opt/module/pulsar/bin/pulsar-daemon stop broker"
echo " --------停止 $i bookKeeper-------"
ssh $i "/opt/module/pulsar/bin/pulsar-daemon stop bookie
echo " --------停止 $i zookeeper-------"
ssh $i "/opt/module/pulsar/bin/pulsar-daemon stop zookeeper
done
};;
esac
添加执行权限 chmod 777 ps.sh