#一、hosts
/etc/hosts
master
----------------------
127.0.0.1 localhost
192.168.64.100 master
192.168.64.101 slave1
192.168.64.102 slave2
192.168.64.103 slave3
192.168.64.104 slave4
-----------------------
slave1
-----------------------
127.0.0.1 localhost
127.0.1.1 slave1
192.168.64.100 master
192.168.64.101 slave1
192.168.64.102 slave2
192.168.64.103 slave3
192.168.64.104 slave4
-----------------------
slave2
-----------------------
127.0.0.1 localhost
127.0.1.1 slave2
192.168.64.100 master
192.168.64.101 slave1
192.168.64.102 slave2
192.168.64.103 slave3
192.168.64.104 slave4
-----------------------
slave3
-----------------------
127.0.0.1 localhost
127.0.1.1 slave3
192.168.64.100 master
192.168.64.101 slave1
192.168.64.102 slave2
192.168.64.103 slave3
192.168.64.104 slave4
-----------------------
slave4
-----------------------
127.0.0.1 localhost
192.168.64.100 master
192.168.64.101 slave1
192.168.64.102 slave2
192.168.64.103 slave3
192.168.64.104 slave4
#二、environment
hadoop-env.sh
-----------------
export JAVA_HOME=/usr/local/jvm/jdk
-----------------
slaves:
-----------------
slave1
slave2
slave3
---------------------
core-site.xml:
--------------------
<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>file:/usr/local/hadoop/tmp</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://master:8020</value>
</property>
</configuration>
-------------------------
hdfs-site.xml:
------------------------
<configuration>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>slave4:50090</value>
</property>
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>file:/usr/local/hadoop/tmp/dfs/name1,file:/usr/local/hadoop/tmp/dfs/name2</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>file:/usr/local/hadoop/tmp/dfs/data1,file:/usr/local/hadoop/tmp/dfs/data2</value>
</property>
</configuration>
----------------------------------------------------------
mapred-site.xml(复制mapred-site.xml.template,再修改文件名)
----------------------------------------------------------
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>Master:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>Master:19888</value>
</property>
</configuration>
----------------------
yarn-site.xml
---------------------
<configuration>
<!-- Site specific YARN configuration properties -->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>master</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
</configuration>
-------------------------
#三、namenode -format
hdfs namenode -format
#四、shell命令
##1、分发复制
xsync
#!/bin/bash
pcount=$#
if (( pcount<1 )); then
echo no args;
exit;
fi
#
p1=$1;
fname=`basename $p1`
#echo fname=$fname;
pdir=`cd -P $(dirname $p1); pwd`
#echo pdir=$pdir
cuser=`whoami`
for(( host=1; host<5;host=host+1)); do
echo ----------------s$host----------
rsync -rvl $pdir/$fname $cuser@slave$host:$pdir
done
rsync xxx
##2、集群操作命令
xcall
#!/bin/bash
pcount=$#
if (( pcount<1 )); then
echo no args;
exit;
fi
#
p1=$1;
fname=`basename $p1`
#echo fname=$fname;
echo --------master--------
$@
cuser=`whoami`
for(( host=1; host<5;host=host+1)); do
echo ---------slave$host--------
ssh slave$host $@
done
xcall rm -rf xxx
xcall jps
…