大数据集群搭建之hadoop-完全分布式

前提 

1、jdk已装好

2、节点间相互免密钥

3、zookeeper已装好

4、时间同步

 

环境-虚拟机centos7 3台    

开始搭建

下载、解压(node01、node02、node03)

tar zxf hadoop-2.7.3 -C /opt/hadoop

cd /opt/hadoop/hadoop-2.7.3/etc/hadoop/

 

 

 vi hadoop-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_162

填自己的javahome路径

 vi mapred-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_162

填自己的javahome路径

 vi yarn-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_162

填自己的javahome路径

 

vi hdfs-site.xml

<configuration>

<property>

  <name>dfs.replication</name>

  <value>2</value>

</property>

<property>

  <name>dfs.nameservices</name>

  <value>mycluster</value>

</property>

<property>

  <name>dfs.ha.namenodes.mycluster</name>

  <value>nn1,nn2</value>

</property>

<property>

  <name>dfs.namenode.rpc-address.mycluster.nn1</name>

  <value>node01:8020</value>

</property>

<property>

  <name>dfs.namenode.rpc-address.mycluster.nn2</name>

  <value>node02:8020</value>

</property>

<property>

  <name>dfs.namenode.http-address.mycluster.nn1</name>

  <value>node01:50070</value>

</property>

<property>

  <name>dfs.namenode.http-address.mycluster.nn2</name>

  <value>node02:50070</value>

</property>

<property>

  <name>dfs.namenode.shared.edits.dir</name>

  <value>qjournal://node01:8485;node02:8485;node03:8485/mycluster</value>

</property>

<property>

  <name>dfs.journalnode.edits.dir</name>

  <value>/var/hadoop/ha/jn</value>

</property>

<property>

  <name>dfs.client.failover.proxy.provider.mycluster</name>

  <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

</property>

<property>

  <name>dfs.ha.fencing.methods</name>

  <value>sshfence</value>

</property>

<property>

  <name>dfs.ha.fencing.ssh.private-key-files</name>

  <value>/root/.ssh/id_rsa</value>

</property>

<property>

   <name>dfs.ha.automatic-failover.enabled</name>

   <value>true</value>

</property>

</configuration>

vi core-site.xml

<configuration>

<property>

  <name>fs.defaultFS</name>

  <value>hdfs://mycluster</value>

</property>

<property>

  <name>hadoop.tmp.dir</name>

  <value>/var/hadoop/ha</value>

</property>

<property>

   <name>ha.zookeeper.quorum</name>

   <value>node01:2181,node02:2181,node03:2181</value>

</property>

</configuration>

配置mapreduce on yarn

vi mapred-site.xml

<property>

        <name>mapreduce.framework.name</name>

        <value>yarn</value>

  </property>

vi yarn-site.xml

<property>

        <name>yarn.nodemanager.aux-services</name>

        <value>mapreduce_shuffle</value>

    </property>

<property>

   <name>yarn.resourcemanager.ha.enabled</name>

   <value>true</value>

</property>

<property>

   <name>yarn.resourcemanager.cluster-id</name>

   <value>cluster1</value>

</property>

<property>

   <name>yarn.resourcemanager.ha.rm-ids</name>

   <value>rm1,rm2</value>

</property>

<property>

   <name>yarn.resourcemanager.hostname.rm1</name>

   <value>node02</value>

</property>

<property>

   <name>yarn.resourcemanager.hostname.rm2</name>

   <value>node03</value>

</property>

<property>

   <name>yarn.resourcemanager.zk-address</name>

   <value>node01:2181,node02:2181,node03:2181</value>

</property>

vi slaves

node01

node02

node03

启动zookeeper

[root@node01 hadoop]# zkServer.sh start

[root@node02 hadoop]# zkServer.sh start

[root@node03 hadoop]# zkServer.sh start

启动journalnode

[root@node01 hadoop]# hadoop-daemon.sh start journalnode

[root@node02 hadoop]# hadoop-daemon.sh start journalnode

[root@node03 hadoop]# hadoop-daemon.sh start journalnode

 

格式化（/var/hadoop下就被创建了一个full的目录）

hdfs namenode -format

主节点启动namenode

[root@node01 current]# hadoop-daemon.sh start namenode

备节点同步主节点元数据

[root@node02 hadoop]# hdfs namenode -bootstrapStandby

格式化zookeeper

[root@node01 current]# hdfs zkfc -formatZK

启动hdfs

[root@node01 current]# start-dfs.sh

[root@node02]# yarn-daemon.sh start resourcemanager

[root@node03]# yarn-daemon.sh start resourcemanager

 

之后每次的启动顺序

启动zookeeper

zkServer.sh start(node01 node02 node03)

启动hdfs

[root@node01 ~]# start-dfs.sh

启动yarn

[root@node01 ~]# start-yarn.sh                               

[root@node02 ~]# yarn-daemon.sh start resourcemanager

[root@node03 ~]# yarn-daemon.sh start resourcemanager