docker部署hadoop-ha集群配置(所有环境已经封装好了)

docker-compose启动方式(docker run方式懒得写了 自己去钻研)

这里先奉上docker hub仓库 地址 docker hub地址超链接

先配置docker-compose.yml文件(端口配置自己看)

version: "3"

services:
  hadoop01:
    image: yujiangxian/hadoop-ha-zk:master
    container_name: master
    privileged: true
    command: ['/usr/sbin/init']
    hostname: master
    tty: true
    stdin_open: true
    ports:
      - 50070:50070
      - 9870:9870
      - 9000:9000
      - 8088:8088
      - 19888:19888
      - "23793:22"
    networks:
      net:
        ipv4_address: 172.18.0.12

  hadoop02:
    image: yujiangxian/hadoop-ha-zk:slave1
    container_name: slave1
    privileged: true
    command: ['/usr/sbin/init']
    hostname: slave1
    tty: true
    stdin_open: true
    ports:
      - "23795:22"
    networks:
      net:
        ipv4_address: 172.18.0.13
      
  hadoop03:
    image: yujiangxian/hadoop-ha-zk:slave2
    container_name: slave2
    privileged: true
    command: ['/usr/sbin/init']
    tty: true
    stdin_open: true
    ports:
      - "23797:22"
    hostname: slave2
    networks:
      net:
        ipv4_address: 172.18.0.14

networks:
  net:
    driver: bridge
    ipam:
      config:
        - subnet: 172.18.0.0/24
          gateway: 172.18.0.1

接下来不用多说,老规矩,进入命令行docker-compose.yml所在路径

docker-compose up -d 
进入容器 source /root/.bash_profile 初始化环境

开启/关闭所有集群服务的脚本已经写好

all-start.sh	所有服务启动
all-stop.sh   	所有服务关闭
allzkstart.sh zookeeper集群启动
allzkstop.sh  zookeeper集群关闭

Hadoop-HA部署配置文件参考

path:/root/.bash_profile

ZOOKEEPER_HOME=/usr/local/src/zk
PATH=$PATH:$ZOOKEEPER_HOME/bin
HADOOP_HOME=/usr/local/src/hadoop
PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

zk配置(本配置默认为环境变量已修改 如果未修改 还请自行修改)

创建数据目录和日志目录

mkdir /usr/local/src/zk/data

mkdir /usr/local/src/zk/logs

复制配置文件

cp zoo_sample.cfg zoo.cfg

修改配置文件

#add host and  port
server.1=master:2888:3888
server.2=master:2888:3888
server.3=master:2888:3888
dataLogDir=/usr/local/src/zk/logs
dataDir=/usr/local/src/zk/data

配置myid 配置文件

echo "1" > /usr/local/src/zk/data/myid

最后分发环境配置文件和zk配置文件

scp 分发 命令自己写

修改myid

echo "2" > /usr/local/src/zk/data/myid
echo "3" > /usr/local/src/zk/data/myid

启动zk(三台)

zkServer.sh start

hadoop-HA配置文件

hadoop-env.sh

#set-java存放路径
export JAVA_HOME=/usr/local/src/jdk

core-site.xml

#指定HDFSnameservice服务名称为ns,与hdfs-site配置相同
<property>
	<name>fs.defaultFS</name>
	<value>hdfs://ns</value>
</property>
#set-指定hadoop临时文件存储路径
<property>
	<name>hadoop.tmp.dir</name>
	<value>/usr/local/src/hadoop/tmp</value>
</property>
#指定zookeeper地址和端口号配置HA需要
<property>
	<name>ha.zookeeper.quorum</name>
	<value>master:2181,slave1:2181,slave2:2181</value>
</property>

hdfs-site.xml

#指定HDFSnameservice服务名ns
<property>
	<name>dfs.nameservices</name>
	<value>ns</value>
</property>
#指定有两个namenode为nn1,nn2
<property>
	<name>dfs.ha.namenode.ns</name>
	<value>nn1,nn2</value>
</property>
#指定nn1 rpc 通信地址
<property>
	<name>dfs.namenode.rpc-address.ns.nn1</name>
	<value>master:9000</value>
</property>
#指定nn1 http通信地址
<property>
	<name>dfs.namenode.http-address.ns.nn1</name>
	<value>master:50070</value>
</property>
#指定nn2 rpc 通信地址
<property>
	<name>dfs.namenode.rpc-address.ns.nn2</name>
	<value>slave1:9000</value>
</property>
#指定nn2 http通信地址
<property>
	<name>dfs.namenode.http-address.ns.nn2</name>
	<value>slave1:50070</value>
</property>
#指定journal地址和端口号
<property>
	<name>dfs.namenode.shared.edits.dir</name>
	<value>qjournal://master:8485;slave1:8485;slave2;8485/ns</value>
</property>
#指定namenode元数据在journalnode上的存放位置
<property>
	<name>dfs.journalnode.edits.dir</name>
	<value>/usr/local/src/hadoop/journal</value>
</property>
#开启HA自动切换
<property>
	<name>dfs.ha.automatic-failover.enabled</name>
	<value>true</value>
</property>
#设置失败自动切换方式
<property>
	<name>dfs.client.failover.proxy.provider.ns</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
#设置隔离机制 ssh会因为无权限访问导致自动切换失败,此时直接使用shell
<property>
<name>dfs.ha.fencing.methods</name>
<value>shell(/bin/true)</value>
</property>
#设置使用ssh隔离时需要ssh免密登录
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value/root/.ssh/id_rsa></value>
</property>
#指定namenode存储空间
<property>
<name>dfs.namenode.name.dir</name>
<value>/usr/local/src/hadoop/tmp/dfs/name</value>
</property>
#指定datanode存储空间
<property>
<name>dfs.datanode.data.dir</name>
<value>/usr/local/src/hadoop/tmp/dfs/data</value>
</property>
#指定数据块副本数
<property>
<name>dfs.replication</name>
<value>3</value>
</property>

mapred-site

#指定mapreduce计算框架使用yarn
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>

yarn-site

#开启yarn-ha
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>


#指定两个resourcemanager的名称为rm1和rm2
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>


#指定rm1主机为master
<property>
<name>yarn.re sourcemanager.hostname.rm1</name>
<value>master</value>
</property>


#指定rm2主机为slave1
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>slave1</value>
</property>


#开启yarn恢复机制
<property>
<name>yarn.resourcemanager.recovery.enabled</name>
<value>true</value>
</property>


#执行resourcemanager恢复机制实现类
<property>
<name>yarn.resourcemanager.store.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
</property>


#配置zookeeper的地址端口
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>master:2181,slave1:2181,slave1:2181</value>
</property>


#指定yarn HA的名称为yarn-ha
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>yarn-ha</value>
</property>


#指定yarn的actice resourcemanager地址
<property>
<name>yarn.resourcemanager.hostname</name>
<value>master</value>
</property>

#指定nodemanager获取数据方式
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>

/usr/local/src/hadoop/etc/hadoop/masters

master
slave1

/usr/local/src/hadoop/etc/hadoop/slaves

master
slave1
slave2

hadoop数据目录

/usr/local/src/hadoop/dfs/name
/usr/local/src/hadoop/dfs/data
/usr/local/src/hadoop/tmp
/usr/local/src/hadoop/journal

集群启动

sh all-start.sh

集群关闭

sh all-stop.sh

最后 BUG记录

hadoop-daemons.sh start journalnode报错

需要which命令 而docker是没有which的

所以需要

 yum -y install which
 ssh slave1 " yum -y install which"
  ssh slave2 " yum -y install which"