黑猴子的家：Docker 搭建Hadoop完全分布式

1、拉取centos基础镜像

[root@hadoop108 ~]# docker pull centos:7.5.1804

2、创建centos-jdk:1.0 镜像

[root@hadoop108 software]# mkdir docker-jdk
[root@hadoop108 software]# cd docker-jdk
[root@hadoop108 docker-jdk]# touch Dockerfile
[root@hadoop108 docker-jdk]# vim Dockerfile
FROM centos:7.5.1804
RUN mkdir -p /opt/software
RUN mkdir -p /opt/module
COPY jdk-8u144-linux-x64.tar.gz /opt/software/
RUN tar -xzvf /opt/software/jdk-8u144-linux-x64.tar.gz -C /opt/module
RUN rm -rf /opt/software/jdk-8u144-linux-x64.tar.gz
ENV JAVA_HOME=/opt/module/jdk1.8.0_144
ENV PATH=$PATH:$JAVA_HOME/bin
[root@hadoop108 docker-jdk]# docker build -t centos-jdk:1.0 ./
[root@hadoop108 docker-jdk]# docker images

3、创建 centos-jdk-ssh:1.0:1.0 镜像

[root@hadoop108 software]# mkdir docker-jdk-ssh
[root@hadoop108 software]# cd docker-jdk-ssh
[root@hadoop108 docker-jdk-ssh]# touch Dockerfile
[root@hadoop108 docker-jdk-ssh]# vim Dockerfile
FROM centos_java8:1.0
MAINTAINER AlexMK liufengji@aliyun.com
RUN curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
RUN sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
RUN yum makecache
RUN yum install -y openssh-server openssh-clients vim net-tools
RUN sed -i '/^HostKey/'d /etc/ssh/sshd_config
RUN echo 'HostKey /etc/ssh/ssh_host_rsa_key'>>/etc/ssh/sshd_config
RUN ssh-keygen -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key
RUN echo 'root:000000' | chpasswd
EXPOSE 22
# 开放所有端口
#EXPOSE 22-65535
RUN mkdir -p /opt
RUN echo '#!/bin/bash' >> /opt/run.sh
RUN echo '/usr/sbin/sshd -D' >> /opt/run.sh
RUN chmod +x /opt/run.sh
CMD ["/opt/run.sh"]
[root@hadoop108 docker-jdk-ssh]# docker build -t centos-jdk-ssh:1.0 ./

4、运行容器

[root@hadoop108 ~]# docker run --name hadoop102 -h hadoop102 -d centos-jdk-ssh:1.0
[root@hadoop108 ~]# docker run --name hadoop103 -h hadoop103 -d centos-jdk-ssh:1.0
[root@hadoop108 ~]# docker run --name hadoop104 -h hadoop104 -d centos-jdk-ssh:1.0

5、查看容器网络

[root@hadoop108 ~]# docker network inspect bridge

6、通过这种方式是可以(不推荐)

[root@hadoop108 ~]# docker exec -it hadoop102 /bin/bash
[root@hadoop108 ~]# docker exec -it hadoop103 /bin/bash
[root@hadoop108 ~]# docker exec -it hadoop104 /bin/bash

7、通过端口映射的方式(不推荐)

[root@hadoop108 ~]# docker run --name hadoop105 -h hadoop105 -p 522:22 -d centos-jdk-ssh:1.0

8、安装工具

[root@hadoop108 ~]# yum install -y bridge-utils git
[root@hadoop108 ~]# git clone https://github.com/jpetazzo/pipework.git
[root@hadoop108 ~]# cp pipework/pipework /usr/bin/

9、配置网络

[root@hadoop108 ~]# cd /usr/local/bin/
[root@hadoop108 bin]# touch docker_network.sh
[root@hadoop108 bin]# chmod 755 docker_network.sh
[root@hadoop108 bin]# vim docker_network.sh
#!/bin/bash

docker start hadoop102
docker start hadoop103
docker start hadoop104

brctl addbr br0; \
ip link set dev br0 up; \
ip addr del 192.168.2.108/24 dev ens33; \
ip addr add 192.168.2.108/24 dev br0; \
brctl addif br0 ens33; \
ip route add default via 192.168.2.2 dev br0

sleep 5

pipework br0 hadoop102 192.168.2.102/24@192.168.2.2
pipework br0 hadoop103 192.168.2.103/24@192.168.2.2
pipework br0 hadoop104 192.168.2.104/24@192.168.2.2
[root@hadoop108 ~]# docker_network.sh

10、配置主机和ip的映射

[root@hadoop102 ~]# vim /etc/hosts
192.168.2.102 hadoop102
192.168.2.103 hadoop103
192.168.2.104 hadoop104

[root@hadoop103 ~]# vim /etc/hosts
192.168.2.102 hadoop102
192.168.2.103 hadoop103
192.168.2.104 hadoop104

[root@hadoop104 ~]# vim /etc/hosts
192.168.2.102 hadoop102
192.168.2.103 hadoop103
192.168.2.104 hadoop104

11、ssh 免密登陆

[root@hadoop102 ~]# ssh-kengen -t rsa
[root@hadoop102 ~]# cd ~/.ssh
[root@hadoop102 .ssh]# ssh-copy-id hadoop102
[root@hadoop102 .ssh]# ssh-copy-id hadoop103
[root@hadoop102 .ssh]# ssh-copy-id hadoop104

[root@hadoop103 ~]# ssh-kengen -t rsa
[root@hadoop103~]# cd ~/.ssh
[root@hadoop103 .ssh]# ssh-copy-id hadoop102
[root@hadoop103 .ssh]# ssh-copy-id hadoop103
[root@hadoop103 .ssh]# ssh-copy-id hadoop104

[root@hadoop104 ~]# ssh-kengen -t rsa
[root@hadoop104 ~]# cd ~/.ssh
[root@hadoop104 .ssh]# ssh-copy-id hadoop102
[root@hadoop104 .ssh]# ssh-copy-id hadoop103
[root@hadoop104 .ssh]# ssh-copy-id hadoop104

12、配置jdk环境变量

[root@hadoop102 ~]# vim  /etc/profile
# JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_144
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin
[root@hadoop102 ~]# source /etc/profile
[root@hadoop102 ~]# java -version

[root@hadoop103 ~]# vim  /etc/profile
# JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_144
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin
[root@hadoop103 ~]# source /etc/profile
[root@hadoop103 ~]# java -version

[root@hadoop104 ~]# vim  /etc/profile
# JAVA_HOME
export JAVA_HOME=/opt/module/jdk1.8.0_144
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin
[root@hadoop104 ~]# source /etc/profile
[root@hadoop104 ~]# java -version

13、安装部署hadoop

102 103 104
hadoop-env.sh
yarn-env.sh
mapred-env.sh
core-site.xml
hdfs-site.xml
yarn-site.xml
mapred-site.xml
slaves

14、格式化hadoop

[root@hadoop102 hadoop-2.7.2]# bin/hdfs namenode -format

15、启动hadoop

[root@hadoop102 hadoop-2.7.2]# sbin/start-dfs.sh
[root@hadoop103 hadoop-2.7.2]# sbin/start-yarn.sh

16、浏览器访问

192.168.2.102:50070
192.168.2.103:8088