centos hadoop集群搭建所需命令记录

1.解压 --  tar -xvf xxx.tar.gz

2.重命名 -- mv  xxx（原）  xxx（新）

3. 删除 openJDK -- rpm -qa|grep java(查看)   rpm -e --nodeps xxx（删除）

4.分发 -- scp -r  /usr/local/hadoop-2.6.1 centosx:/usr/local

5.修改文件夹所属用户组 -- chown hadoop.hadoop /usr/local/hadoop-2.6.1

6.环境配置 --

vim /etc/profile
JAVA_HOME=/usr/local/java/jdk1.8.0_221
HADOOP_HOME=/usr/local/hadoop-2.6.1
SPARK_HOME=/usr/local/spark-2.2.1
CLASSPATH=.:$JAVA_HOME/lib.tools.jar
PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$PATH
export JAVA_HOME CLASSPATH PATH
source /etc/profile

7.修改文件夹权限 -- chmod 777 /usr/local/java（开放所有权限）

8. 关于vim编辑器的使用 -- i(进入编辑模式)  Esc(退出编辑模式)  :wq(退出编辑器)

9.hdfs namenode -format

10.hadoop dfsadmin -report

11.配置文件 -- 

core-site.xml

<property>
		<name>fs.defaultFS</name>
		<value>hdfs://centos1:9000</value>
	</property>
	<property>
		<name>hadoop.tmp.dir</name>
		<value>file:/usr/local/hadoop-2.6.1/tmp</value>
   	</property>
	<property>
		<name>io.file.buffer.size</name>
		<value>131072</value>
	</property>
	<property>
		<name>hadoop.proxyuser.hduser.hosts</name>
		<value>*</value>
	</property>
	<property>
		<name>hadoop.proxyuser.hduser.groups</name>
		<value>*</value>
	</property>

hdfs-site.xml

<property>
		<name>dfs.namenode.secondary.http-address</name>
		<value>centos1:9001</value>
	</property>
	 <property>
            <name>dfs.replication</name>
            <value>2</value>
        </property>
	<property>
             <name>dfs.namenode.name.dir</name> 
             <value>file:/usr/local/hadoop-2.6.1/dfs/name</value>
	</property>
	<property>
             <name>dfs.datanode.data.dir</name>
             <value>file:/usr/local/hadoop-2.6.1/dfs/data</value>
	</property>
	<property>
		<name>dfs.webhdfs.enabled</name>
		<value>true</value>
	</property>
	<property>
		<name>dfs.permissions</name>
		<value>false</value>
	</property>
	<property>
		<name>heartbeat.recheck.interval</name>
		<value>180000</value>
	</property>
	<property>
		<name>dfs.heartbeat.interval</name>
		<value>3</value>
	</property>

map-red-site.xml(复制命令：cp mapred-site.xml.template mapred-site.xml )

<property>
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
	</property>
	<property>
		<name>mapreduce.jobhistory.address</name>
		<value>centos1:10020</value>
	</property>
	<property>
		<name>mapreduce.jobhistory.webapp.address</name>
		<value>centos1:19888</value>
	</property>
	<property>
 		<name>mapred.task.timeout</name>
 		<value>600000</value>
 		<final>true</final>
	</property>
	<property>
		<name>mapreduce.map.memory.mb</name>
		<value>2048</value>
	</property>
	<property>
		<name>mapreduce.reduce.memory.mb</name>
		<value>2048</value>
	</property>
	<property>
		<name>mapreduce.jobhistory.joblist.cache.size</name>
		<value>2000</value>
	</property>

yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>
	<property>
        	<name>yarn.resourcemanager.hostname</name>
        	<value>centos1</value>
    	</property>
	<property>
		<name>yarn.nodemanager.aux-services</name>
		<value>mapreduce_shuffle</value>
	</property>
	<property>
		<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
		<value>org.apache.hadoop.mapred.ShuffleHandler</value>
	</property>
	<property>
		<name>yarn.resourcemanager.scheduler.class</name>
		<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
	</property>
	<property>
		<name>yarn.resourcemanager.address</name>
		<value>centos1:8032</value>
	</property>
	<property>
		<name>yarn.resourcemanager.scheduler.address</name>
		<value>centos1:8030</value>
	</property>
	<property>
		<name>yarn.resourcemanager.resource-tracker.address</name>
		<value>centos1:8031</value>
	</property>
	<property>
		<name>yarn.resourcemanager.admin.address</name>
		<value>centos1:8033</value>
	</property>
	<property>
		<name>yarn.resourcemanager.webapp.address</name>
		<value>centos1:8088</value>
	</property>
	<property>
		<name>yarn.log-aggregation-enable</name>
		<value>true</value>
	</property>
	<property>
		<name>yarn.log.server.url</name>
		<value>http://centos1:19888/jobhistory/logs</value>
	</property>
	<property> 
    		<name>yarn.nodemanager.resource.memory-mb</name>  
    		<value>8192</value>  
	</property>
	<property> 
    		<name>yarn.scheduler.minimum-allocation-mb</name>  
    		<value>2048</value>  
	</property>
	<property>  
    		<name>yarn.scheduler.maximum-allocation-mb</name>  
    		<value>8192</value>  
	</property>
	<property>
             <name>yarn.resourcemanager.scheduler.class</name>
            <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
     	</property>
	<property>
        	<name>yarn.log-aggregation.retain-seconds</name>
	        <value>604800</value>
    	</property>
	<property>
		<name>yarn.nodemanager.remote-app-log-dir</name>
		<value>/tmp/logs</value>
	</property>
	<property>
	    	<name>yarn.nodemanager.vmem-pmem-ratio</name>
	    	<value>2.1</value>
	</property>
</configuration>

备注：如果磁盘空间不够大 将data 目录改到 home下