centos7+hadoop2.8.1集群安装

环境要求：centos7   三台虚拟机       

软件要求：jdk1.8     hadoop-2.8.1.tar.gz  

  

//添加用户hadoop ，组名为admin

useradd  -g admin -s /bin/bash  -m  hadoop   

       

//更改密码最小长度为6

authconfig  --passminlen=6  --update   

 

//设置hadoop用户密码                 

passwd  hadoop   

//修改/etc/sysconfig/network 三个虚拟机分别设置Master Slave1 Slave2

NETWORKING=yes

HOSTNAME=Master

 

//使配置文件立即生效

source  /etc/sysconfig/network

        

//修改/etc/hosts文件， 三个虚拟机都添加

10.4.93.183  Master

10.4.94.127  Slave1

 10.4.94.228  Slave2     

//切换到hadoop用户

su – hadoop

 

//在hadoop用户根目录添加目录  三个虚拟机都添加

mkdir .ssh

 

//进入.ssh目录删除之前的ssh文件

rm id_rsa.pub

 

//设置ssh无密码登录  输入命令，全部回车

ssh-keygen –t rsa

 

//将pub文件内容复制到authorized_keys

 cat id_rsa.pub >>./authorized_keys

 

//将pub文件复制到另外两个虚拟机

 scp id_rsa.pub hadoop@Slave1:/home/hadoop/.ssh

 scp id_rsa.pub hadoop@Slave2:/home/hadoop/.ssh

 

 //进入另外两个虚拟机/home/hadoop/.ssh目录 执行如下命令

 cat id_rsa.pub >>./authorized_keys

 

.ssh目录的权限必须是700 

.ssh/authorized_keys文件权限必须是600

赋权语句：chmod 700 /home/hadoop/.ssh

//第一个机器的ssh无密码登录设置完成，另外两个虚拟机从上述步骤的创建.ssh目录开始，重复上述步骤

                                                

 

//配置JDK环境变量 /home/hadoop/.bashrc

export JAVA_HOME=/home/hadoop/java

export PATH=$JAVA_HOME/bin:$PATH

 

//使文件生效

source  .bashrc

 

//关闭防火墙

systemctl stop firewalld.service

systemctl disable firewalld.service

 

//上传hadoop压缩包到Master机器（hadoop用户）

tar –xvf hadoop-2.8.1.tar.gz

//重命名为hadoop

mv  /home/hadoop/hadoop-2.8.1  /home/hadoop/hadoop

 

//添加环境变量 三个虚拟机都需要添加  /home/hadoop/.baserc

export HADOOP_HOME=/home/hadoop/hadoop

export PATH=$HADOOP_HOME/bin:$PATH

 

//修改/home/hadoop/hadoop/etc/hadoop/slaves文件，master机器添加两行  删除localhost

 Slave1

 Slave2

 

//修改core-site.xml

<configuration>

        <property>

                 <name>fs.defaultFS</name>

                <value>hdfs://Master:9000</value>

        </property>

        <property>

                <name>hadoop.tmp.dir</name>

                <value>file:/home/hadoop/hadoop/tmp</value>

                <description>Abase forother temporary directories.</description>

        </property>

</configuration>

 

//修改hdfs-site.xml

 <configuration>

        <property>

                <name>dfs.namenode.secondary.http-address</name>

                <value>Master:50090</value>

        </property>

        <property>

                <name>dfs.replication</name>

                <value>1</value>

        </property>

        <property>

                <name>dfs.namenode.name.dir</name>

                <value>file:/home/hadoop/hadoop/tmp/dfs/name</value>

        </property>

        <property>

                <name>dfs.datanode.data.dir</name>

                <value>file:/home/hadoop/hadoop/tmp/dfs/data</value>

        </property>

 </configuration>

 

 //修改mapred-site.xml

 <configuration>

        <property>

                <name>mapreduce.framework.name</name>

                <value>yarn</value>

        </property>

        <property>

                <name>mapreduce.jobhistory.address</name>

                <value>Master:10020</value>

        </property>

        <property>

                <name>mapreduce.jobhistory.webapp.address</name>

                <value>Master:19888</value>

        </property>

</configuration>

 

//修改yarn-site.xml

<configuration>

        <property>

                <name>yarn.resourcemanager.hostname</name>

                <value>Master</value>

        </property>

        <property>

                <name>yarn.nodemanager.aux-services</name>

                <value>mapreduce_shuffle</value>

        </property>

</configuration>

 

//将hadoop打成tar包，拷贝到Slave1Slave2

tar –cvf hadoop.tar hadoop

scp hadoop.tar hadoop@Slave1:/home/hadoop

scp hadoop.tar hadoop@Slave2:/home/hadoop

 

//登录Slave1,解压tar包

tar –xvf hadoop.tar

 

//登录Slave2,解压tar包

tar –xvf hadoop.tar

 

//登录Master执行初始化

hdfs namenode –format

 

//切换到目录/home/hadoop/hadoop/sbin启动hadoop

./start-dfs.sh

./start-yarn.sh

./mr-jobhistory-daemon.sh start historyserver

 

//三个机器分别查看进程

 jps

 

//Master机器进程

2896 SecondaryNameNode

3365 JobHistoryServer

2743 NameNode

3447 Jps

3049 ResourceManager

 

//Slave1机器进程

2677 Jps

2536 NodeManager

2458 DataNode

 

//Slave2机器进程

2449 DataNode

2679 Jps

2526 NodeManager

 

//关闭hadoop集群

./stop-yarn.sh

./stop-dfs.sh

mr-jobhistory-daemon.sh stop historyserver