NIFI相关环境搭建

NIFI相关环境搭建

1. 部署docker

1. 安装gcc

   yum -y install gcc
   yum -y install gcc-c++
   

2. 安装yum-utils包，提供yum-config-manager实用程序，并设置稳定存储库

   yum install -y yum-utils
   
   yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
   

3. 更新yum软件包索引

   yum makecache fast
   

4. 安装最新版本docker

   yum install docker-ce docker-ce-cli containerd.io
   

5. 启动docker

   systemctl start docker
   

6. 验证docker是否安装正确

   docker run hello-world
   

7. 配置阿里云镜像加速器

   # 创建daemon.json文件
   vim /etc/docker/daemon.json
   # 在daemon.json中写入
   {
   	"registry-mirrors": ["https://自己的账号加速信息.mirror.aliyuncs.com"]
   }
   # 加载daemon文件，并重启docker
   systemctl daemon-reload
   systemctl restart docker
   

8. 验证镜像加速器是否生效

   # 查看docker信息，查看Registry Mirrors属性
   docker info
   

2. 部署docker-compose

1. 安装python环境

   yum -y install epel-release
   
   yum install python3 python3-setuptools python3-pip python3-ldap python3-devel gcc gcc-c++ -y
   
   pip3 install --timeout=3600 Pillow pylibmc captcha jinja2 sqlalchemy==1.3.8 psd-tools django-pylibmc django-simple-captcha
   

2. 安装docker-compose

   cd /usr/local/bin/
   wget https://github.com/docker/compose/releases/download/1.14.0-rc2/docker-compose-Linux-x86_64
   rename docker-compose-Linux-x86_64 docker-compose docker-compose-Linux-x86_64
   chmod +x /usr/local/bin/docker-compose
   

3. 部署NIFI

docker pull apache/nifi:1.12.0

docker run -it --name nifi -v /opt/nifi/lib:/opt/nifi/lib -p 8443:8443 -d -e NIFI_WEB_HTTP_PORT='8443'  apache/nifi:1.12.0

# web 访问
http://192.168.201.5:8443/nifi/

4. 部署kafka

1. docker-compose.yml 放到/opt 目录下

   version: '2'
   services:
     zookeeper:
       image: zookeeper
       volumes:
         - /opt/zookeeper:/data
       ports:
         - "2181:2181"
          
     kafka:
       image: wurstmeister/kafka
       ports:
         - "9092:9092"
       environment:
         KAFKA_ADVERTISED_HOST_NAME: 192.168.201.5
         KAFKA_MESSAGE_MAX_BYTES: 2000000  # kafka(message.max.bytes) 会接收单个消息size的最大限制，默认值为1000000 , ≈1M
         KAFKA_CREATE_TOPICS: "topic_for_nifi:3:1" # 初始创建的topics，可以不设置, topic名称:partition数:replicas数
         KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
       volumes:
         - ./kafka-logs:/kafka
         - /opt/kafka/docker.sock:/var/run/docker.sock
    
     kafka-manager:
       image: kafkamanager/kafka-manager
       ports:
         - 9020:9000
       environment:
         ZK_HOSTS: zookeeper:2181
   

2. 启动

   docker-compose up -d