CentOS7下安装hadoop

固定IP

   修改文件  /etc/sysconfig/network-scripts/ifcfg-ens33

   下面内容为参考

   TYPE=Ethernet

   BOOTPROTO=static

   NM_CONTROLLED=yes

   #--------------下面内容根据vmware配置相应变更

   IPADDR=192.168.41.199

   NETMASK=255.255.255.0

   GATEWAY=192.168.41.2

   #--------------

   DNS1=8.8.8.8

   DNS2=114.114.114.114

   NAME=ens33

   DEVICE=ens33

   ONBOOT=yes

   重启网络服务  systemctl start network 或 service network restart

   查看IP地址  ip addr

4. 设置IP地址映射

  vi /etc/hosts

  添加Hadoop节点的映射信息

  192.168.41.151 master

  192.168.41.152 slave1

  192.168.41.153 slave2

5. 选择性安装 yum -y install vim ntp

6. 关闭防火墙

  #service iptables stop & chkconfig iptables off

  systemctl stop firewalld    

  systemctl disable firewalld

  禁用SELinux  vi /etc/selinux/config  将SELINUX设置为disabled,重启系统后生效

   大数据培训

7. 修改hostname

  vi /etc/hostname

  或者 hostnamectl set-hostname master(主机名称)

8. 安装java1.8版本

   tar -zxvf /opt/jdk-8u162-linux-x64.tar.gz -C ./

   

  在 /etc/profile中添加

  export JAVA_HOME=/usr/java/jdk1.8.0_162

  export PATH=$PATH:$JAVA_HOME/bin:.

   

  执行 source /etc/profile 让配置生效

  执行 java -version验证java安装正确

   

9. poweroff停止虚拟机

  然后克隆出slave1和slave2

   

10. 设置slave1和slave2的IP地址

   vi /etc/sysconfig/network-scripts/ifcfg-ens33

   

11. 设置slave1和slave2的hostname

   hostnamectl set-hostname  xxxx

   

12. 设置master机器的ntp配置(可选)

13. 设置slave1和slave2的ntp配置(可选)

14. 在master上配置SSH无密码登录

   （1）使用ssh-keygen产生公钥与私钥对。

   输入命令“ssh-keygen -t rsa”，接着按三次Enter键

   （2）用ssh-copy-id将公钥复制到远程机器中

   ssh-copy-id -i /root/.ssh/id_rsa.pub master//依次输入yes,123456（root用户的密码）

   ssh-copy-id -i /root/.ssh/id_rsa.pub slave1

   ssh-copy-id -i /root/.ssh/id_rsa.pub slave2

   （3）验证是否设置无密码登录

   依次输入

   ssh slave1

   ssh slave2

15. 安装hadoop

   上传 安装包  后解压 tar -zxvf /opt/hadoop-2.7.1.tar.gz -C /usr/local

   15.3. 配置Hadoop

   进入目录：

   cd /usr/local/hadoop-2.7.1/etc/hadoop/

   依次修改下面的文件：

   15.4.1 core-site.xml

   <configuration>

       <property>

       <name>fs.defaultFS</name>  

         <value>hdfs://master:8020</value>    

         </property>  

       <property>

         <name>hadoop.tmp.dir</name>

         <value>/var/log/hadoop/tmp</value> //生成日志目录

       </property>

   </configuration>

   15.4.2 hadoop-env.sh

   export JAVA_HOME=/usr/java/jdk1.8.0_151

   电脑中自己的路径：export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_162

   15.4.3 hdfs-site.xml

   <configuration>

   <property>

       <name>dfs.namenode.name.dir</name>

       <value>file:///data/hadoop/hdfs/name</value>

   </property>

   <property>

       <name>dfs.datanode.data.dir</name>

       <value>file:///data/hadoop/hdfs/data</value>

   </property>

   <property>

        <name>dfs.namenode.secondary.http-address</name>

        <value>master:50090</value>

   </property>

   <property>  //备份数，缺省为3份

        <name>dfs.replication</name>

        <value>3</value>

   </property>

   </configuration>

   15.4.4 mapred-site.xml

   复制cp mapred-site.xml.template mapred-site.xml

   <configuration>

   <property>

       <name>mapreduce.framework.name</name>

       <value>yarn</value>   //配置yarn框架

   </property>

   <!-- jobhistory properties -->

   <property>

       <name>mapreduce.jobhistory.address</name>

       <value>master:10020</value>

   </property>

   <property>

        <name>mapreduce.jobhistory.webapp.address</name>

        <value>master:19888</value>

   </property>

   </configuration>

   15.4.5 yarn-site.xml //yarn框架

   <configuration>

     <property>

       <name>yarn.resourcemanager.hostname</name>

       <value>master</value>

     </property>    

     <property>

       <name>yarn.resourcemanager.address</name>

       <value>${yarn.resourcemanager.hostname}:8032</value>

     </property>

     <property>

       <name>yarn.resourcemanager.scheduler.address</name>

       <value>${yarn.resourcemanager.hostname}:8030</value>

     </property>

     <property>

       <name>yarn.resourcemanager.webapp.address</name>

       <value>${yarn.resourcemanager.hostname}:8088</value>

     </property>

     <property>

       <name>yarn.resourcemanager.webapp.https.address</name>

       <value>${yarn.resourcemanager.hostname}:8090</value>

     </property>

     <property>

       <name>yarn.resourcemanager.resource-tracker.address</name>

       <value>${yarn.resourcemanager.hostname}:8031</value>

     </property>

     <property>

       <name>yarn.resourcemanager.admin.address</name>

       <value>${yarn.resourcemanager.hostname}:8033</value>

     </property>

     <property>

       <name>yarn.nodemanager.local-dirs</name>

       <value>/data/hadoop/yarn/local</value>

     </property>

     <property>

       <name>yarn.log-aggregation-enable</name>

       <value>true</value>

     </property>

     <property>

       <name>yarn.nodemanager.remote-app-log-dir</name>

       <value>/data/tmp/logs</value>

     </property>

   <property>  

    <name>yarn.log.server.url</name>  

    <value>http://master:19888/jobhistory/logs/</value>

    <description>URL for job history server</description>

   </property>

   <property>

      <name>yarn.nodemanager.vmem-check-enabled</name>

       <value>false</value>

     </property>

    <property>

       <name>yarn.nodemanager.aux-services</name>

       <value>mapreduce_shuffle</value>

     </property>

     <property>

       <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>

         <value>org.apache.hadoop.mapred.ShuffleHandler</value>

         </property>

   <property>  

           <name>yarn.nodemanager.resource.memory-mb</name>  

           <value>2048</value>  

    </property>  

    <property>  

           <name>yarn.scheduler.minimum-allocation-mb</name>  

           <value>512</value>  

    </property>    

    <property>  

           <name>yarn.scheduler.maximum-allocation-mb</name>  

           <value>4096</value>  

    </property>  

    <property>  

       <name>mapreduce.map.memory.mb</name>  

       <value>2048</value>  

    </property>  

    <property>  

       <name>mapreduce.reduce.memory.mb</name>  

       <value>2048</value>  

    </property>  

    <property>  

       <name>yarn.nodemanager.resource.cpu-vcores</name>  

       <value>1</value>  

    </property>

   </configuration>

   15.4.6 yarn-env.sh

   export JAVA_HOME=/usr/java/jdk1.8.0_151

   15.4.7 slaves

   删除localhost，添加：

   slave1

   slave2

   拷贝hadoop安装文件到集群slave节点

   scp -r /usr/local/hadoop-2.7.1 slave1:/usr/local/

   scp -r /usr/local/hadoop-2.7.1 slave2:/usr/local/

   15.5.在/etc/profile添加Hadoop路径

   export HADOOP_HOME=/usr/local/hadoop-2.7.1

   export PATH=$HADOOP_HOME/bin:$PATH

   source /etc/profile使修改生效

   15.6. 格式化NameNode

   进入目录

   cd /usr/local/hadoop-2.7.1/bin

   执行格式化

    ./hdfs namenode -format

   15.7.启动集群

   进入目录

   cd /usr/local/hadoop-2.7.1/sbin

   执行启动：

   ./start-dfs.sh

   ./start-yarn.sh

   ./mr-jobhistory-daemon.sh start historyserver

   使用jps，查看进程

   [root@master sbin]# jps

   1765 NameNode

   1929 SecondaryNameNode

   2378 JobHistoryServer

   2412 Jps

   2077 ResourceManager

   [root@slave1 ~]# jps

   1844 Jps

   1612 DataNode

   1711 NodeManager

   在Windows下C:\Windows\System32\drivers\etc\hosts添加IP映射

   192.168.41.151 master

   15.8. 浏览器查看：

   http://master:50070

   http://master:8088