DOCKER集群安装SQOOP&FLUME
前言
已经配好hadoop-hdfs集群，集群由三个docker容器构成，一个namenode，两个datanode，本次安装FLUME&SQOOP，将FLUME安装在hdfs集群的namenode节点，SQOOP&HIVE置于另一容器，MYSQL节点也置于另一容器，搭建好的集群共5个的docker容器，通过docker网络连接。
DOCKER集群架构图
#创建一个docker网络
docker network create nanzi_bigdata_network
#查看nanzi_bigdata_network细节以及各个容器的ip
docker network inspect nanzi_bigdata_network
注： hdfs集群的三台容器已经在nanzi_bigdata_network网络内，后续容器间通信需要查看节点ip即通过该方法
各组件版本号
hadoop 3.2.0
flume 1.9.0
sqoop 1.4.7
mysql 5.7.42
namenode节点安装flume
1.安装flume
apt update & apt install wget
wget https://archive.apache.org/dist/flume/1.9.0/apache-flume-1.9.0-bin.tar.gz
#将下载下来的apache-flume-1.9.0-bin.tar.gz解压到/opt下
tar zxvf apache-flume-1.9.0-bin.tar.gz /opt
#修改文件名，方便后续操作
mv /opt/apache-flume-1.9.0-bin /opt/flume
rm apache-flume-1.9.0-bin.tar.gz
2.配置环境变量
vim ~/.bashrc
#添加
export FLUME_HOME=/opt/flume
export PATH=$PATH:$FLUME_HOME/bin
#刷新配置文件
source ~/.bashrc
检查是否配置成功
flume-ng version
3.修改 flume-env.sh，添加JAVA_HOME路径
#进入flume/conf/下
cp flume-env.sh.template ./flume-env.sh
#修改flume-env.sh
vim flume-env.sh
#添加
export JAVA_HOME=/....
4.测试flume
在/opt下创建一个flume-config用于放置flume任务的配置文件
mkdir /opt/flume-config
#写一个测试案例
vim flume-example.conf
"""
docker.sinks = logSink
docker.sources = netcatSource
docker.channels = inMemoryChannel
docker.sources.netcatSource.type = netcat
docker.sources.netcatSource.bind = 0.0.0.0
docker.sources.netcatSource.port = 44444
docker.sources.netcatSource.channels = inMemoryChannel
docker.channels.inMemoryChannel.type = memory
docker.channels.inMemoryChannel.capacity = 999
docker.channels.inMemoryChannel.transactionCapacity = 99
docker.sinks.logSink.type = logger
docker.sinks.logSink.channel = inMemoryChannel
"""
#启动flume Application 监听
flume-ng agent -c /opt/flume/conf -f /opt/flume-config/flume-example.conf -n docker -Dflume.root.logger=INFO,console
#开启另一终端
nc localhost 44444
可以看到，已经成功监听。
测试flume连接hdfs
#写一个测试案例
vim flume-hdfs.conf
"""
a1.sources = r1
a1.sinks = k1
a1.channels = c1
a1.sources.r1.type = exec
a1.sources.r1.command = tail -F /opt/flume-logs/test.txt //监听/opt/flume-logs/test.txt，当该文件改变时flume会监听到
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://master:9000/flume/logs
a1.sinks.k1.hdfs.filePrefix = log-
a1.sinks.k1.hdfs.fileType = DataStream
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.channels.c1.type = memory
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
"""
#启动flume Application 监听
flume-ng agent -c /opt/flume/conf -f /opt/flume-config/flume-hdfs.conf -n a1 -Dflume.root.logger=INFO,console
#开启另一终端向/opt/flume-logs/test.txt内写入内容
echo "nanzi lucky!" >> /opt/flume-logs/test.txt
#检查内容是否一致
hadoop fs -cat /flume/logs/*
可以看见，flume已经将监听到的内容成功上传至hdfs。
新容器内安装SQOOP
1.安装mysql
#使用mysql:5.7.42镜像
docker pull mysql:5.7.42
#创建挂载目录
mkdir -p /opt/mysql/conf /opt/mysql/data
#在/opt/mysql/conf下编写MYSQL的配置文件my.conf
vim my.conf
"""
[client]
default-character-set = utf8mb4
[mysqld]
datadir = /opt/mysql/data
character_set_server = utf8mb4
collation_server = utf8mb4_bin
secure-file-priv= NULL
symbolic-links=0
!includedir /etc/mysql/conf.d/
"""
#创建容器实例
docker run --restart=unless-stopped -d --name nanzi_mysql -v /opt/mysql/conf/my.cnf:/etc/mysql/my.cnf -v /opt/mysql/data:/var/lib/mysql -p 3307:3306 -e MYSQL_ROOT_PASSWORD=123456 mysql:5.7.42
#查看容器id
docker ps -a | grep nanzi_mysql
#启动容器
docker start 8735a53bb782(容器id)
#进入容器内部
docker exec -it 8735a53bb782 bash
#登入mysql
mysql -u root -p
1.1将mysql节点容器加入nanzi_bigdata_network网络
docker network connect nanzi_bigdata_network 8735a53bb782(容器id)
2.安装sqoop
2.1下载sqoop
注： 官方集成的hadoop包停留在了2.6.0版本,在hadoop3.2.0版本会提示类版本过低错误，但纯净版sqoop有缺少必须的第三方库,所以将这两个包下载下来，提取部分sqoop_hadoop2.6.0版本的jar包放到纯净版sqoop的lib目录下，在sqoop配置文件中加入获取当前环境中的hadoop的lib库来使用.
#使用ubuntu:latest镜像
docker pull ubuntu:latest
#创建容器实例
docker run -it --name nanzi_sqoop ubuntu:latest
#下载sqoop1.4.7
apt update & apt install wget
wget https://archive.apache.org/dist/sqoop/1.4.7/sqoop-1.4.7.tar.gz
wget https://archive.apache.org/dist/sqoop/1.4.7/sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz
#将下载下来的sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz和sqoop-1.4.7.tar.gz解压到/opt下
tar zxvf sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz /opt
tar zxvf sqoop-1.4.7.tar.gz /opt
#修改文件名，方便后续操作
mv /opt/sqoop-1.4.7.tar.gz /opt/sqoop
#提取sqoop-1.4.7.bin__hadoop-2.6.0中的ant-contrib-1.0b3.jar & ant-eclipse-1.0-jvm1.2.jar & avro-1.8.1.jar 三个jar包
cp /opt/sqoop-1.4.7.bin__hadoop-2.6.0/lib/ant-contrib-1.0b3.jar ant-eclipse-1.0-jvm1.2.jar avro-1.8.1.jar /opt/sqoop/lib/
#删除不需要的文件及压缩包
rm -r sqoop-1.4.7.bin__hadoop-2.6.0
rm sqoop-1.4.7.tar.gz
rm sqoop-1.4.7.bin__hadoop-2.6.0.tar.gz
2.2 配置环境变量
vim ~/.bashrc
#添加
export SQOOP_HOME=/opt/SQOOP
export JAVA_HOME=.....(你自己的路径,没有jdk的自行下载)
export PATH=$PATH:$SQOOP_HOME/bin
#刷新配置文件
source ~/.bashrc
检查是否配置成功
sqoop version
2.3 将jdbc传入sqoop/lib下，因为使用的是mysql5.7.42,所以使用mysql-connector-java-5.1.40连接mysql
#下载mysql-connector-java-5.1.40
wget https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-java-5.1.40.zip
#解压至 /opt
tar zxvf mysql-connector-java-5.1.40.zip /opt
#将mysql-connector-java-5.1.40内的mysql-connector-java-5.1.40-bin.jar移至 /opt/sqoop/lib下
cp /opt/mysql-connector-java-5.1.40/mysql-connector-java-5.1.40-bin.jar /opt/sqoop/lib
2.4 由于sqoop需要使用到hadoop的jar包，而使用的ubuntu没有安装hadoop，所以先在容器内安装hadoop，hadoop版本与hdfs集群内相同
#下载hadoop-3.2.4.tar.gz
wget https://dlcdn.apache.org/hadoop/common/hadoop-3.2.4/hadoop-3.2.4.tar.gz
#将hadoop-3.2.4.tar.gz解压到/opt下
tar zxvf hadoop-3.2.4.tar.gz /opt
将hdfs集群中主节点下的hadoop配置文件拷贝到当前hadoop下的配置文件中
2.5 修改 sqoop-env.sh 添加 HADOOP_HOME路径
#进入sqoop/conf/下
cp sqoop-env-template.sh ./sqoop-env.sh
#修改sqoop-env.sh
vim sqoop-env.sh
#添加
export HADOOP_COMMON_HOME=/opt/hadoop(填写自己的路径)
export HADOOP_MAPRED_HOME=/opt/hadoop
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH
2.6 将sqoop节点容器加入nanzi_bigdata_network网络
docker network connect nanzi_bigdata_network 2ab626c63ddd(容器id)
3.测试sqoop
#使用sqoop列出mysql中所有的表
sqoop list-databases --username root --password '123456' --connect jdbc:mysql://172.19.0.6:3306/ //172.19.0.6为mysql节点容器ip，查看方法见前言
#将mysql内的表通过sqoop传入hdfs
#首先在mysql建立测试表
CREATE TABLE balabala (
ID INT PRIMARY KEY,
NAME VARCHAR(50),
AGE INT
);
INSERT INTO balabala (ID, NAME, AGE) VALUES
(1, 'John', 25),
(2, 'Alice', 30),
(3, 'Bob', 22);
#启动sqoop任务
sqoop import --connect 'jdbc:mysql://172.19.0.6:3306/test_sqoop?useSSL=false&allowPublicKeyRetrieval=true' \
--username root \
--password 123456 \
--table balabala \
--target-dir hdfs://master:9000/sqoop_test \
-m 1
#hdfs://master:9000 为hdfs的路径，master可以写hdfs集群的namenode节点的ip，或者可以在hadoop的core-site.xml 进行配置即可写配置的名称
测试成功!!!
#将hdfs的表通过sqoop导入mysql
#首先在mysql建立测试表，用于接受数据
CREATE TABLE hualahuala (
ID INT PRIMARY KEY,
NAME VARCHAR(50),
AGE INT
);
#将上面传入hdfs的内容导回mysql
sqoop export \
--connect jdbc:mysql://172.19.0.6:3306/test_sqoop?useSSL=false \
--username root \
--password 123456 \
--table hualahuala \
--input-fields-terminated-by ',' \
--export-dir sqoop_test/part-m-00000
测试成功!!!数据已经导入mysql。
6646
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
