Hadoop2.7.5双主基本配置（记录）

hadoop集群搭建首先需要JAVA环境，还有要zookeeper集群，可以参考JAVA和zookeeper安装文档

准备

软件准备

请到hadoop官网下载hadoop2.7.5版本：https://hadoop.apache.org/releases.html，可以下载binary包直接安装hadoop-2.7.5.tar.gz，或者下载source包编译安装

环境准备

服务器列表：
10.242.154.185 master001
10.242.154.186 master002
10.242.154.187 slave001
10.242.154.188 slave002
10.242.154.189 slave003
10.242.154.190 slave004
10.242.154.191 slave005

zookeeper:
10.242.167.200
10.242.167.201
10.242.167.202

安装

配置SSH免密登录，可以参考脚本
生产秘钥命令：ssh-keygen -t rsa
配置相关文件：
配置core-site.xml

<configuration>
        <property>
                <name>fs.defaultFS</name>
                <value>hdfs://master001</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/data/tmp</value>
        </property>
        <property>
                <name>ha.zookeeper.quorum</name>
                <value>10.242.167.200:2181,10.242.167.201:2181,10.242.167.202:2181</value>
        </property>
        <property>
                <name>hadoop.proxyuser.root.groups</name>
                <value>root</value>
         </property>
         <property>
                <name>hadoop.proxyuser.root.hosts</name>
                <value>*</value>
         </property>
</configuration>

配置hadoop-env.sh

基本配置
export HADOOP_LOG_DIR=/log/hadoop/
export JAVA_HOME=/usr/local/jdk1.8.0_121

配置hdfs-site.xml

<configuration>
        <property>
                <name>dfs.nameservices</name>
                <value>master001</value>
        </property>
        <property>
                <name>dfs.ha.namenodes.master001</name>
                <value>master001,master002</value>
        </property>
        <property>
                <name>dfs.namenode.rpc-address.master001.master001</name>
                <value>master001:8020</value>
        </property>
        <property>
                <name>dfs.namenode.http-address.master001.master001</name>
                <value>master001:50070</value>
        </property>
        <property>
                <name>dfs.namenode.rpc-address.master001.master002</name>
                <value>master002:8020</value>
        </property>
        <property>
                <name>dfs.namenode.http-address.master001.master002</name>
                <value>master002:50070</value>
        </property>
        <property>
                <name>dfs.namenode.shared.edits.dir</name>
                <value>qjournal://slave003:8485;slave002:8485;slave01:8485/master002</value>
        </property>
        <property>
                <name>dfs.journalnode.edits.dir</name>
                <value>/usr/local/hadoop-2.7.5/journal</value>
        </property>
        <property>
                <name>dfs.ha.automatic-failover.enabled</name>
                <value>true</value>
        </property>
        <property>
            <name>dfs.permissions</name>
            <value>false</value>
        </property>
        <property>
                <name>dfs.client.failover.proxy.provider.masters</name>
                <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
        </property>
        <property>
                <name>dfs.ha.fencing.methods</name>
                <value>
                        sshfence
                        shell(/bin/true)
                </value>
        </property>
        <property>
                <name>dfs.ha.fencing.ssh.private-key-files</name>
                <value>/home/hadoop/.ssh/id_rsa</value>
        </property>
        <property>
                <name>dfs.ha.fencing.ssh.connect-timeout</name>
                <value>30000</value>
        </property>
        <property>
                <name>dfs.hosts.exclude</name>
                <value>/usr/local/hadoop/etc/hadoop/excludes</value> 
        </property>
</configuration>

配置mapred-site.xml

<configuration>
	<property>
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
	</property>
	<property>
		<name>mapreduce.map.memory.mb</name>
		<value>4096</value>
	</property>	
	<property>
		<name>mapreduce.reduce.memory.mb</name>
		<value>4096</value>
	</property>	
	<property>
		<name>yarn.app.mapreduce.am.resource.mb</name>
		<value>4096</value>
	</property>	
	<property>
		<name>mapreduce.task.io.sort.mb</name>
		<value>1536</value>
	</property>	
	<property>
		<name>mapreduce.map.java.opts</name>
		<value>-Xmx3072m</value>
	</property>	
	<property>
		<name>mapreduce.reduce.java.opts</name>
		<value>-Xmx3072m</value>
	</property>	
	<property>
		<name>yarn.app.mapreduce.am.command-opts</name>
		<value>-Xmx3072m</value>
	</property>		
</configuration>

配置slaves

slave001
slave002
slave003
slave004
slave005

配置yarn-site.xml

<configuration>
        <property>
                <name>yarn.resourcemanager.ha.enabled</name>
                <value>true</value>
        </property>
        <property>
                <name>yarn.resourcemanager.cluster-id</name>
                <value>RM_HA_ID</value>
        </property>
        <property>
                <name>yarn.resourcemanager.ha.rm-ids</name>
                <value>rm1,rm2</value>
        </property>
        <property>
                <name>yarn.resourcemanager.hostname.rm1</name>
                <value>master001</value>
        </property>
        <property>
                <name>yarn.resourcemanager.hostname.rm2</name>
                <value>master002</value>
        </property>
        <property>
                <name>yarn.resourcemanager.recovery.enabled</name>
                <value>true</value>
        </property>
        <property>
                <name>yarn.resourcemanager.store.class</name>
                <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value>
        </property>
        <property>
                <name>yarn.resourcemanager.zk-address</name>
                <value>10.242.167.200:2181,10.242.167.201:2181,10.242.167.202:2181</value>
        </property>
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
        <property>
                <name>yarn.scheduler.minimum-allocation-mb</name>
                <value>4096</value>
        </property>
        <property>
                <name>yarn.scheduler.maximum-allocation-mb</name>
                <value>12288</value>
        </property>		
        <property>
                <name>yarn.nodemanager.resource.memory-mb</name>
                <value>12288</value>
        </property>				
</configuration>

然后在各个节点建好相关目录
将/usr/local/hadoop 整个目录scp到其他节点
然后进入/usr/local/hadoop/sbin
执行`./start-all.sh`
启动集群

访问WEB端
http://10.242.154.186:50070
active

standby
http://10.242.154.185:50070

基本搭建结束