普通安装
准备工作
安装jdk
上传jdk到linux中,并执行解压命令解压缩安装包
tar -zxzf jdk1.8.0.tar.gz -C /opt/module
添加环境变量
vim /etc/profile #添加环境变量
##JAVA_HOME## export JAVA_HOME=/opt/module/jdk1.8.0 export PATH=$PATH:$JAVA_HOME/bin
source /etc/profile ###刷新环境变量 java -version #查看java版本
安装zookeeper集群
规划 在hadoop102、hadoop103和hadoop104三个节点上部署Zookeeper。
上传安装包到linux中,并执行解压缩命令
tar -zxvf zookeeper-3.4.10.tar.gz -C /opt/module/ cd /opt/module/zookeeper-3.4.10 #进入zk目录 mkdir -p zkData #创建日志目录 cd conf #进入配置目录 mv zoo_sample.cfg zoo.cfg #修改默认配置文件名
配置zoo.cfg
dataDir=/opt/module/zookeeper-3.4.10/zkData
增加如下配置
#######################cluster########################## server.2=hadoop102:2888:3888 #hadoop102、hadoop103、hadoop104为主机名,并在 server.3=hadoop103:2888:3888 #hosts中做了域名映射,此处为hadoop是用的hadoop环 server.4=hadoop104:2888:3888 #境
添加集群编号
cd /opt/module/zookeeper-3.4.10/zkData touch myid #创建并修改文件 vim myid #在里面写上对应的编号,如 2
拷贝配置好的zookeeper到其他机器
scp -r zookeeper-3.4.10/ root@hadoop103:$PWD scp -r zookeeper-3.4.10/ root@hadoop104:$PWD
修改 myid
vim myid #修改成相对应的ip编号即可
启动集群
[root@hadoop102 zookeeper-3.4.10]# bin/zkServer.sh start [root@hadoop103 zookeeper-3.4.10]# bin/zkServer.sh start [root@hadoop104 zookeeper-3.4.10]# bin/zkServer.sh start
kafka安装
解压缩安装包并改名
tar -zxvf kafka_2.11-0.11.0.0.tgz -C /opt/module/ mv kafka_2.11-0.11.0.0 ./kafka cd kafka
修改配置文件
vi server.properties ##########输入一下内容############ #broker的全局唯一编号,不能重复 broker.id=0 #删除topic功能使能 delete.topic.enable=true #处理网络请求的线程数量 num.network.threads=3 #用来处理磁盘IO的现成数量 num.io.threads=8 #发送套接字的缓冲区大小 socket.send.buffer.bytes=102400 #接收套接字的缓冲区大小 socket.receive.buffer.bytes=102400 #请求套接字的缓冲区大小 socket.request.max.bytes=104857600 #kafka运行日志存放的路径 log.dirs=/opt/module/kafka/logs #topic在当前broker上的分区个数 num.partitions=1 #用来恢复和清理data下数据的线程数量 num.recovery.threads.per.data.dir=1 #segment文件保留的最长时间,超时将被删除 log.retention.hours=168 #配置连接Zookeeper集群地址 zookeeper.connect=hadoop102:2181,hadoop103:2181,hadoop104:2181
配置环境变量
vi /etc/profile
#KAFKA_HOME export KAFKA_HOME=/opt/module/kafka export PATH=$PATH:$KAFKA_HOME/bin
source /etc/profile
分发安装包既即环境变量
scp -r /opt/module/kafka/ root@hadoop104:$PWD scp -r /opt/module/kafka/ root@hadoop103:$PWD scp -r /etc/profile root@hadoop103:$PWD scp -r /etc/profile root@hadoop104:$PWD
修改/opt/module/kafka/config/server.properties中的broker.id=1、broker.id=2。注:broker.id不得重复
Docker安装Kafka
编写zookeeper.Dockerfile文件
FROM centos:6.6
RUN yum -y install vim lsof wget tar bzip2 unzip vim-enhanced passwd sudo yum-utils hostname net-tools rsync man git make automake cmake patch logrotate python-devel libpng-devel libjpeg-devel pwgen python-pip
RUN mkdir /opt/java &&\
wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u102-b14/jdk-8u102-linux-x64.tar.gz -P /opt/java
RUN tar zxvf /opt/java/jdk-8u102-linux-x64.tar.gz -C /opt/java &&\
JAVA_HOME=/opt/java/jdk1.8.0_102 &&\
sed -i "/^PATH/i export JAVA_HOME=$JAVA_HOME" /root/.bash_profile &&\
sed -i "s%^PATH.*$%&:$JAVA_HOME/bin%g" /root/.bash_profile &&\
source /root/.bash_profile
ENV ZOOKEEPER_VERSION "3.4.6"
RUN mkdir /opt/zookeeper &&\
wget http://mirror.olnevhost.net/pub/apache/zookeeper/zookeeper-$ZOOKEEPER_VERSION/zookeeper-$ZOOKEEPER_VERSION.tar.gz -P /opt/zookeeper
RUN tar zxvf /opt/zookeeper/zookeeper*.tar.gz -C /opt/zookeeper
RUN echo "source /root/.bash_profile" > /opt/zookeeper/start.sh &&\
echo "cp /opt/zookeeper/zookeeper-"$ZOOKEEPER_VERSION"/conf/zoo_sample.cfg /opt/zookeeper/zookeeper-"$ZOOKEEPER_VERSION"/conf/zoo.cfg" >> /opt/zookeeper/start.sh &&\
echo "/opt/zookeeper/zookeeper-$"ZOOKEEPER_VERSION"/bin/zkServer.sh start-foreground" >> /opt/zookeeper/start.sh
EXPOSE 2181
ENTRYPOINT ["sh", "/opt/zookeeper/start.sh"]
编写kafka.Dockerfile文件
FROM centos:6.6
ENV KAFKA_VERSION "0.8.2.2"
RUN yum -y install vim lsof wget tar bzip2 unzip vim-enhanced passwd sudo yum-utils hostname net-tools rsync man git make automake cmake patch logrotate python-devel libpng-devel libjpeg-devel pwgen python-pip
RUN mkdir /opt/java &&\
wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u102-b14/jdk-8u102-linux-x64.tar.gz -P /opt/java
RUN mkdir /opt/kafka &&\
wget http://apache.fayea.com/kafka/$KAFKA_VERSION/kafka_2.11-$KAFKA_VERSION.tgz -P /opt/kafka
RUN tar zxvf /opt/java/jdk-8u102-linux-x64.tar.gz -C /opt/java &&\
JAVA_HOME=/opt/java/jdk1.8.0_102 &&\
sed -i "/^PATH/i export JAVA_HOME=$JAVA_HOME" /root/.bash_profile &&\
sed -i "s%^PATH.*$%&:$JAVA_HOME/bin%g" /root/.bash_profile &&\
source /root/.bash_profile
RUN tar zxvf /opt/kafka/kafka*.tgz -C /opt/kafka &&\
sed -i 's/num.partitions.*$/num.partitions=3/g' /opt/kafka/kafka_2.11-$KAFKA_VERSION/config/server.properties
RUN echo "source /root/.bash_profile" > /opt/kafka/start.sh &&\
echo "cd /opt/kafka/kafka_2.11-"$KAFKA_VERSION >> /opt/kafka/start.sh &&\
echo "sed -i 's%zookeeper.connect=.*$%zookeeper.connect=zookeeper:2181%g' /opt/kafka/kafka_2.11-"$KAFKA_VERSION"/config/server.properties" >> /opt/kafka/start.sh &&\
echo "bin/kafka-server-start.sh config/server.properties" >> /opt/kafka/start.sh &&\
chmod a+x /opt/kafka/start.sh
EXPOSE 9092
ENTRYPOINT ["sh", "/opt/kafka/start.sh"]
创建zookeeper镜像
docker -t jason/zookeeper:3.4.6 -f zookeeper.Dockerfile .
创建kafka镜像
docker -t jason/kafka:0.8.2.2 -f kafka.Dockerfile .
查看当前镜像目录
docker images|grepo jason
启动zookeeper
docker run -itd --name zookeeper -h zookeeper -p2181:2181 jason/zookeeper:3.4.6 bash
docker ps -a #查看运行过的images状态
docker ps #查看正在运行的images列表
启动kafka
docker run -itd --name kafka -h kafka -p9092:9092 --link jason/kafka:0.8.2.2 bash
## --link 将kafka链接到zookeeper
docker ps #查看正在运行的images列表
lsof -i:9092 #查看端口监听
lsof -i:2181
进入kafka的controller里面
docker exec -it kafka bash
在docker里面测试创建topic
source /root/.bash_profile #刷新
cd /opt/kafka/kafka_2.11-0.8.2.2
bin/kafka-topic.sh --create --topic test1 --zookeeper zookeeper:2181 --partitions 3 --replication-factor 1
bin/kafka-topic.sh --create --topic test2 --zookeeper zookeeper:2181 --partitions 3 --replication-factor 1 #创建topic
bin/kafka-topic.sh --list --topic test2 --zookeeper zookeeper:2181 ##查看