Windows PC上创建大数据职业技能竞赛实验环境之三--Spark、Hive、Flume、Kafka和Flink环境的搭建

在前述hadoop-base基础容器环境的基础上，实现Spark、Hive、Flume、kafka和Flink实验环境的搭建。我们已将前述的hadoop-base基础容器进行可阶段的保存：

sudo docker commit "hadoop base v3" hadoop-base centos/hadoop-base:v3

现在，如果已经将前述作业的hadoop-base容器停用并删除，用保存的centos/hadoop-base:v3镜像重新开启hadoop-base容器。加入共享目录，暴露ssh端口，我们将原22端口映射为22222。

sudo docker run -it --name hadoop-base -v /home/wslu/data:/root/data -p 22222:22 centos/hadoop-base:v3 /bin/bash

1 安装Spark、Hive、Zookeeper、Flume、Kafka和Flink软件包

方法类似，即解压软件包，移动到/root/hadoop目录下，配置环境变量。

Spark

[root@f9ea223c4f75 data]# ls
apache-echarts-5.1.0-src.zip   flink-1.10.2-bin-scala_2.11.tgz             redis-4.0.1.tar.gz
apache-echarts-5.1.2-src.zip   hadoop-2.7.7.tar.gz                         scala-2.11.12.tgz
apache-flume-1.7.0-src.tar.gz  kafka_2.11-2.0.0.tgz                        spark-2.1.1-bin-hadoop2.7.tgz
apache-hive-2.3.4-bin.tar.gz   mysql-5.7.36-linux-glibc2.12-x86_64.tar.gz  vue.global.js

[root@f9ea223c4f75 data]# tar xzvf spark-2.1.1-bin-hadoop2.7.tgz

[root@f9ea223c4f75 data]# mv spark-2.1.1-bin-hadoop2.7 /root/hadoop/spark-2.1.1-bin-hadoop2.7
[root@f9ea223c4f75 ~]# ls /root/hadoop/spark-2.1.1-bin-hadoop2.7
bin  conf  data  examples  jars  LICENSE  licenses  NOTICE  python  R  README.md  RELEASE  sbin  yarn

[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc

JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin

export JAVA_HOME JRE_HOME CLASSPATH PATH
export HADOOP_HOME SPARK_HOME

 Hive

[root@f9ea223c4f75 ~]# cd data
[root@f9ea223c4f75 data]# tar zxvf apache-hive-2.3.4-bin.tar.gz

[root@f9ea223c4f75 data]# mv apache-hive-2.3.4-bin /root/hadoop/apache-hive-2.3.4-bin

[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# ls hadoop/apache-hive-2.3.4-bin
bin  binary-package-licenses  conf  examples  hcatalog  jdbc  lib  LICENSE  NOTICE  RELEASE_NOTES.txt  scripts
[root@f9ea223c4f75 ~]#

[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc

JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin
HIVE_HOME=/root/hadoop/apache-hive-2.3.4-bin
PATH=$PATH:$HIVE_HOME/bin

export JAVA_HOME JRE_HOME CLASSPATH PATH
export HADOOP_HOME SPARK_HOME HIVE_HOME

Zookeeper 

[root@f9ea223c4f75 ~]# cd data
[root@f9ea223c4f75 data]# tar zxvf apache-zookeeper-3.6.3-bin.tar.gz

[root@f9ea223c4f75 data]# mv apache-zookeeper-3.6.3-bin /root/hadoop/apache-zookeeper-3.6.3-bin

[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# ls hadoop/apache-zookeeper-3.6.3-bin
bin  conf  docs  lib  LICENSE.txt  NOTICE.txt  README.md  README_packaging.md
[root@f9ea223c4f75 ~]#

[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc

JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin
HIVE_HOME=/root/hadoop/apache-hive-2.3.4-bin
PATH=$PATH:$HIVE_HOME/bin
ZOOKEEPER_HOME=/root/hadoop/apache-zookeeper-2.6.3-bin
PATH=$PATH:$ZOOKEEPER_HOME/bin

export JAVA_HOME JRE_HOME CLASSPATH PATH
export HADOOP_HOME SPARK_HOME HIVE_HOME ZOOKEEPER_HOME

Flume

[root@f9ea223c4f75 ~]# cd data
[root@f9ea223c4f75 data]# tar zxvf apache-flume-1.7.0-bin.tar.gz

[root@f9ea223c4f75 data]# mv apache-flume-1.7.0-bin /root/hadoop/apache-flume-1.7.0-bin

[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# ls hadoop/apache-flume-1.7.0-bin
bin  CHANGELOG  conf  DEVNOTES  doap_Flume.rdf  docs  lib  LICENSE  NOTICE  README.md  RELEASE-NOTES  tools
[root@f9ea223c4f75 ~]#

[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc

JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin
HIVE_HOME=/root/hadoop/apache-hive-2.3.4-bin
PATH=$PATH:$HIVE_HOME/bin
FLUME_HOME=/root/hadoop/apache-flume-1.7.0-bin
FLUME_CONF_DIR=$FLUME_HOME/conf
PATH=$PATH:$FLUME_HOME/bin
ZOOKEEPER_HOME=/root/hadoop/apache-zookeeper-2.6.3-bin
PATH=$PATH:$ZOOKEEPER_HOME/bin

export JAVA_HOME JRE_HOME CLASSPATH PATH
export HADOOP_HOME SPARK_HOME HIVE_HOME ZOOKEEPER_HOME
export FLUME_HOME FLUME_CONF_DIR

Kafka

[root@f9ea223c4f75 ~]# cd data
[root@f9ea223c4f75 data]# tar zxvf kafka_2.11-2.0.0.tgz

[root@f9ea223c4f75 data]# mv apache-zookeeper-3.6.3-bin /root/hadoop/kafka_2.11-2.0.0

[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# ls hadoop/kafka_2.11-2.0.0
bin  config  libs  LICENSE  NOTICE  site-docs
[root@f9ea223c4f75 ~]#

[root@f9ea223c4f75 ~]# cd
[root@f9ea223c4f75 ~]# vi .bashrc

JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.322.b06-1.el7_9.x86_648
CLASSPATH=.:$JAVA_HOME/jre/lib:$JAVA_HOME/lib:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
PATH=$PATH:$JAVA_HOME/bin
HADOOP_HOME=/root/hadoop/hadoop-2.7.7
PATH=$PATH:$HADOOP_HOME/bin
SPARK_HOME=/root/hadoop/spark-2.1.1-bin-hadoop2.7
PATH=$PATH:$SPARK_HOME/bin
HIVE_HOME=/root/hadoop/apache-hive-2.3.4-bin
PATH=$PATH:$HIVE_HOME/bin
FLUME_HOME=/root/hadoop/apache-flume-1.7.0-bin
FLUME_CONF_DIR=$FLUME_HOME/conf
PATH=$PATH:$FLUME_HOME/bin
ZOOKEEPER_HOME=/root/hadoop/apache-zookeeper-2.6.3-bin
PATH=$PATH:$ZOOKEEPER_HOME/bin
KAFKA_HOME=/root/hadoop/kafka_2.11-2.0.0
PATH=$PATH:$KAFKA_HOME/bin

export JAVA