Kafka集群安装之普通安装和Docker方式安装

普通安装

准备工作

安装jdk
  • 上传jdk到linux中,并执行解压命令解压缩安装包

    tar -zxzf jdk1.8.0.tar.gz -C /opt/module
  • 添加环境变量

    vim /etc/profile   #添加环境变量
    
    ##JAVA_HOME##
    
    export JAVA_HOME=/opt/module/jdk1.8.0
    export PATH=$PATH:$JAVA_HOME/bin
    source /etc/profile         ###刷新环境变量
    java -version               #查看java版本
安装zookeeper集群

规划 在hadoop102、hadoop103和hadoop104三个节点上部署Zookeeper。

  • 上传安装包到linux中,并执行解压缩命令

    tar -zxvf zookeeper-3.4.10.tar.gz -C /opt/module/
    cd /opt/module/zookeeper-3.4.10       #进入zk目录
    mkdir -p zkData       #创建日志目录
    cd conf       #进入配置目录
    mv zoo_sample.cfg  zoo.cfg        #修改默认配置文件名
  • 配置zoo.cfg

    dataDir=/opt/module/zookeeper-3.4.10/zkData

    增加如下配置

    
    #######################cluster##########################
    
    server.2=hadoop102:2888:3888      #hadoop102、hadoop103、hadoop104为主机名,并在
    server.3=hadoop103:2888:3888      #hosts中做了域名映射,此处为hadoop是用的hadoop环
    server.4=hadoop104:2888:3888      #境
  • 添加集群编号

    cd    /opt/module/zookeeper-3.4.10/zkData
    touch myid    #创建并修改文件
    vim myid  #在里面写上对应的编号,如 2
  • 拷贝配置好的zookeeper到其他机器

    scp -r zookeeper-3.4.10/ root@hadoop103:$PWD
    scp -r zookeeper-3.4.10/ root@hadoop104:$PWD
  • 修改 myid

    vim myid  #修改成相对应的ip编号即可
  • 启动集群

    [root@hadoop102 zookeeper-3.4.10]# bin/zkServer.sh start
    [root@hadoop103 zookeeper-3.4.10]# bin/zkServer.sh start
    [root@hadoop104 zookeeper-3.4.10]# bin/zkServer.sh start

    kafka安装

    解压缩安装包并改名

    tar -zxvf kafka_2.11-0.11.0.0.tgz -C /opt/module/
    mv kafka_2.11-0.11.0.0 ./kafka
    cd kafka 

    修改配置文件

    vi server.properties
    
    
    ##########输入一下内容############
    
    
    #broker的全局唯一编号,不能重复
    
    broker.id=0
    
    #删除topic功能使能
    
    delete.topic.enable=true
    
    #处理网络请求的线程数量
    
    num.network.threads=3
    
    #用来处理磁盘IO的现成数量
    
    num.io.threads=8
    
    #发送套接字的缓冲区大小
    
    socket.send.buffer.bytes=102400
    
    #接收套接字的缓冲区大小
    
    socket.receive.buffer.bytes=102400
    
    #请求套接字的缓冲区大小
    
    socket.request.max.bytes=104857600
    
    #kafka运行日志存放的路径
    
    log.dirs=/opt/module/kafka/logs
    
    #topic在当前broker上的分区个数
    
    num.partitions=1
    
    #用来恢复和清理data下数据的线程数量
    
    num.recovery.threads.per.data.dir=1
    
    #segment文件保留的最长时间,超时将被删除
    
    log.retention.hours=168
    
    #配置连接Zookeeper集群地址
    
    zookeeper.connect=hadoop102:2181,hadoop103:2181,hadoop104:2181

    配置环境变量

    vi /etc/profile
    
    #KAFKA_HOME
    
    export KAFKA_HOME=/opt/module/kafka
    export PATH=$PATH:$KAFKA_HOME/bin
    source /etc/profile

    分发安装包既即环境变量

    scp -r /opt/module/kafka/ root@hadoop104:$PWD
    scp -r /opt/module/kafka/ root@hadoop103:$PWD
    scp -r /etc/profile root@hadoop103:$PWD
    scp -r /etc/profile root@hadoop104:$PWD

    修改/opt/module/kafka/config/server.properties中的broker.id=1、broker.id=2。注:broker.id不得重复

Docker安装Kafka

编写zookeeper.Dockerfile文件

FROM centos:6.6

RUN yum -y install vim lsof wget tar bzip2 unzip vim-enhanced passwd sudo yum-utils hostname net-tools rsync man git make automake cmake patch logrotate python-devel libpng-devel libjpeg-devel pwgen python-pip

RUN mkdir /opt/java &&\
    wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u102-b14/jdk-8u102-linux-x64.tar.gz -P /opt/java

RUN tar zxvf /opt/java/jdk-8u102-linux-x64.tar.gz -C /opt/java &&\
    JAVA_HOME=/opt/java/jdk1.8.0_102 &&\
    sed -i "/^PATH/i export JAVA_HOME=$JAVA_HOME" /root/.bash_profile &&\
    sed -i "s%^PATH.*$%&:$JAVA_HOME/bin%g" /root/.bash_profile &&\
    source /root/.bash_profile

ENV ZOOKEEPER_VERSION "3.4.6"

RUN mkdir /opt/zookeeper &&\
    wget http://mirror.olnevhost.net/pub/apache/zookeeper/zookeeper-$ZOOKEEPER_VERSION/zookeeper-$ZOOKEEPER_VERSION.tar.gz -P /opt/zookeeper

RUN tar zxvf /opt/zookeeper/zookeeper*.tar.gz -C /opt/zookeeper

RUN echo "source /root/.bash_profile" > /opt/zookeeper/start.sh &&\
    echo "cp /opt/zookeeper/zookeeper-"$ZOOKEEPER_VERSION"/conf/zoo_sample.cfg /opt/zookeeper/zookeeper-"$ZOOKEEPER_VERSION"/conf/zoo.cfg" >> /opt/zookeeper/start.sh &&\
    echo "/opt/zookeeper/zookeeper-$"ZOOKEEPER_VERSION"/bin/zkServer.sh start-foreground" >> /opt/zookeeper/start.sh

EXPOSE 2181

ENTRYPOINT ["sh", "/opt/zookeeper/start.sh"]

编写kafka.Dockerfile文件

FROM centos:6.6

ENV KAFKA_VERSION "0.8.2.2"

RUN yum -y install vim lsof wget tar bzip2 unzip vim-enhanced passwd sudo yum-utils hostname net-tools rsync man git make automake cmake patch logrotate python-devel libpng-devel libjpeg-devel pwgen python-pip

RUN mkdir /opt/java &&\
    wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/8u102-b14/jdk-8u102-linux-x64.tar.gz -P /opt/java

RUN mkdir /opt/kafka &&\
    wget http://apache.fayea.com/kafka/$KAFKA_VERSION/kafka_2.11-$KAFKA_VERSION.tgz -P /opt/kafka

RUN tar zxvf /opt/java/jdk-8u102-linux-x64.tar.gz -C /opt/java &&\
    JAVA_HOME=/opt/java/jdk1.8.0_102 &&\
    sed -i "/^PATH/i export JAVA_HOME=$JAVA_HOME" /root/.bash_profile &&\
    sed -i "s%^PATH.*$%&:$JAVA_HOME/bin%g" /root/.bash_profile &&\
    source /root/.bash_profile

RUN tar zxvf /opt/kafka/kafka*.tgz -C /opt/kafka &&\
    sed -i 's/num.partitions.*$/num.partitions=3/g' /opt/kafka/kafka_2.11-$KAFKA_VERSION/config/server.properties

RUN echo "source /root/.bash_profile" > /opt/kafka/start.sh &&\
    echo "cd /opt/kafka/kafka_2.11-"$KAFKA_VERSION >> /opt/kafka/start.sh &&\
    echo "sed -i 's%zookeeper.connect=.*$%zookeeper.connect=zookeeper:2181%g'  /opt/kafka/kafka_2.11-"$KAFKA_VERSION"/config/server.properties" >> /opt/kafka/start.sh &&\
    echo "bin/kafka-server-start.sh config/server.properties" >> /opt/kafka/start.sh &&\
    chmod a+x /opt/kafka/start.sh

EXPOSE 9092

ENTRYPOINT ["sh", "/opt/kafka/start.sh"]

创建zookeeper镜像

docker -t jason/zookeeper:3.4.6 -f zookeeper.Dockerfile .

创建kafka镜像

docker -t jason/kafka:0.8.2.2 -f kafka.Dockerfile .

查看当前镜像目录

docker images|grepo jason

启动zookeeper

docker run -itd --name zookeeper -h zookeeper -p2181:2181 jason/zookeeper:3.4.6 bash
docker ps -a #查看运行过的images状态
docker ps    #查看正在运行的images列表

启动kafka

docker run -itd --name kafka -h kafka -p9092:9092 --link jason/kafka:0.8.2.2 bash
## --link 将kafka链接到zookeeper
docker ps    #查看正在运行的images列表
lsof -i:9092    #查看端口监听
lsof -i:2181

进入kafka的controller里面

docker exec -it kafka bash 

在docker里面测试创建topic

source /root/.bash_profile #刷新
cd /opt/kafka/kafka_2.11-0.8.2.2
bin/kafka-topic.sh --create --topic test1 --zookeeper zookeeper:2181 --partitions 3 --replication-factor 1
bin/kafka-topic.sh --create --topic test2 --zookeeper zookeeper:2181 --partitions 3 --replication-factor 1      #创建topic
bin/kafka-topic.sh --list --topic test2 --zookeeper zookeeper:2181 ##查看

没有更多推荐了,返回首页