在前述hadoop-base基础容器环境的基础上,实现Spark、Hive、Flume、kafka和Flink实验环境的搭建。我们已将前述的hadoop-base基础容器进行可阶段的保存:
sudo docker commit "hadoop base v3" hadoop-base centos/hadoop-base:v3
现在,如果已经将前述作业的hadoop-base容器停用并删除,用保存的centos/hadoop-base:v3镜像重新开启hadoop-base容器。加入共享目录,暴露ssh端口,我们将原22端口映射为22222。
sudo docker run -it --name hadoop-base -v /home/wslu/data:/root/data -p 22222:22 centos/hadoop-base:v3 /bin/bash
1 安装Spark、Hive、Zookeeper、Flume、Kafka和Flink软件包
方法类似,即解压软件包,移动到/root/hadoop目录下,配置环境变量。
Spark
[root@f9ea223c4f75 data]# ls
apache-echarts-5.1.0-src.zip flink-1.10.2-bin-scala_2.11.tgz redis-4.0.1.tar.gz
apache-echarts-5.1.2-src.zip hadoop-2.7.7.tar.gz scala-2.11.12.tgz
apache-flume-1.7.0-src.tar.gz kafka_2.11-2.0.0.tgz spark-2.1.1-bin-hadoop2.7.tgz
apache-hive-2.3.4-bin.tar.gz mysql-5.7.36-linux-glibc2.12-x86_64.tar.gz vue.global.js
[root@f9ea223c4f75 data]# tar xzvf spark-2.1.1-bin-hadoop2.7.tgz
[root@f9ea223c4f75 data]# mv spark-2.1.1-bin-hadoop2.7 /root/hadoop/spark-2.1.1-bin-hadoop2.7
[root@f9ea223c4f75 ~]# ls /root/hadoop/spark-2.1.1-bin-hadoop2.7
bin conf data examples jars LICENSE licenses NOTICE python R README.md RELEASE sbin yarn
[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc
JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin
export JAVA_HOME JRE_HOME CLASSPATH PATH
export HADOOP_HOME SPARK_HOME
Hive
[root@f9ea223c4f75 ~]# cd data
[root@f9ea223c4f75 data]# tar zxvf apache-hive-2.3.4-bin.tar.gz
[root@f9ea223c4f75 data]# mv apache-hive-2.3.4-bin /root/hadoop/apache-hive-2.3.4-bin
[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# ls hadoop/apache-hive-2.3.4-bin
bin binary-package-licenses conf examples hcatalog jdbc lib LICENSE NOTICE RELEASE_NOTES.txt scripts
[root@f9ea223c4f75 ~]#
[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc
JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin
HIVE_HOME=/root/hadoop/apache-hive-2.3.4-bin
PATH=$PATH:$HIVE_HOME/bin
export JAVA_HOME JRE_HOME CLASSPATH PATH
export HADOOP_HOME SPARK_HOME HIVE_HOME
Zookeeper
[root@f9ea223c4f75 ~]# cd data
[root@f9ea223c4f75 data]# tar zxvf apache-zookeeper-3.6.3-bin.tar.gz
[root@f9ea223c4f75 data]# mv apache-zookeeper-3.6.3-bin /root/hadoop/apache-zookeeper-3.6.3-bin
[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# ls hadoop/apache-zookeeper-3.6.3-bin
bin conf docs lib LICENSE.txt NOTICE.txt README.md README_packaging.md
[root@f9ea223c4f75 ~]#
[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc
JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin
HIVE_HOME=/root/hadoop/apache-hive-2.3.4-bin
PATH=$PATH:$HIVE_HOME/bin
ZOOKEEPER_HOME=/root/hadoop/apache-zookeeper-2.6.3-bin
PATH=$PATH:$ZOOKEEPER_HOME/bin
export JAVA_HOME JRE_HOME CLASSPATH PATH
export HADOOP_HOME SPARK_HOME HIVE_HOME ZOOKEEPER_HOME
Flume
[root@f9ea223c4f75 ~]# cd data
[root@f9ea223c4f75 data]# tar zxvf apache-flume-1.7.0-bin.tar.gz
[root@f9ea223c4f75 data]# mv apache-flume-1.7.0-bin /root/hadoop/apache-flume-1.7.0-bin
[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# ls hadoop/apache-flume-1.7.0-bin
bin CHANGELOG conf DEVNOTES doap_Flume.rdf docs lib LICENSE NOTICE README.md RELEASE-NOTES tools
[root@f9ea223c4f75 ~]#
[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc
JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin
HIVE_HOME=/root/hadoop/apache-hive-2.3.4-bin
PATH=$PATH:$HIVE_HOME/bin
FLUME_HOME=/root/hadoop/apache-flume-1.7.0-bin
FLUME_CONF_DIR=$FLUME_HOME/conf
PATH=$PATH:$FLUME_HOME/bin
ZOOKEEPER_HOME=/root/hadoop/apache-zookeeper-2.6.3-bin
PATH=$PATH:$ZOOKEEPER_HOME/bin
export JAVA_HOME JRE_HOME CLASSPATH PATH
export HADOOP_HOME SPARK_HOME HIVE_HOME ZOOKEEPER_HOME
export FLUME_HOME FLUME_CONF_DIR
Kafka
[root@f9ea223c4f75 ~]# cd data
[root@f9ea223c4f75 data]# tar zxvf kafka_2.11-2.0.0.tgz
[root@f9ea223c4f75 data]# mv apache-zookeeper-3.6.3-bin /root/hadoop/kafka_2.11-2.0.0
[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# ls hadoop/kafka_2.11-2.0.0
bin config libs LICENSE NOTICE site-docs
[root@f9ea223c4f75 ~]#
[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc
JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin
HIVE_HOME=/root/hadoop/apache-hive-2.3.4-bin
PATH=$PATH:$HIVE_HOME/bin
FLUME_HOME=/root/hadoop/apache-flume-1.7.0-bin
FLUME_CONF_DIR=$FLUME_HOME/conf
PATH=$PATH:$FLUME_HOME/bin
ZOOKEEPER_HOME=/root/hadoop/apache-zookeeper-2.6.3-bin
PATH=$PATH:$ZOOKEEPER_HOME/bin
KAFKA_HOME=/root/hadoop/kafka_2.11-