centos hadoop-2.3.0（HA+Federation+YARN）配置

最新推荐文章于 2023-02-19 18:19:55 发布

天然呆的技术博客

最新推荐文章于 2023-02-19 18:19:55 发布

阅读量827

点赞数

分类专栏： hadoop-2.x

本文链接：https://blog.csdn.net/u013494310/article/details/38017731

版权

hadoop-2.x 专栏收录该内容

14 篇文章 0 订阅

订阅专栏

scp -r mwd@gs-server-v-127:~/hadoop-2.3.0 mwd@gs-server-v-129:~/

新建用户组和用户：

sudo adduser mwd

sudo passwd mwd
sudo vi /etc/sudoers
mwd ALL=(ALL) NOPASSWD: ALL

sudo yum install svn
sudo yum install autoconfautomakelibtoolcmake
sudo yum install ncurses-devel
sudo yum install openssl-devel
sudo yum install gcc*
6.安装maven
wget http://apache.fayea.com/apache-mirror/maven/maven-3/3.2.1/binaries/apache-maven-3.2.1-bin.tar.gz
tar xzvf./apache-maven-3.2.1-bin.tar.gz
mv./ apache-maven-3.2.1 /usr/local

安装jdk
http://www.oracle.com/technetwork/java/javase/downloads/jdk7-downloads-1880260.html

配置ssh免密码访问

每台节点上分别执行

ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys

在128机器操作
scp mwd@gs-server-v-127:~/.ssh/id_rsa.pub ~/.ssh/127_rsa.pub
cat ~/.ssh/127_rsa.pub >> ~/.ssh/authorized_keys
scp mwd@gs-server-v-128:~/.ssh/id_rsa.pub ~/.ssh/128_rsa.pub
cat ~/.ssh/128_rsa.pub >> ~/.ssh/authorized_keys
scp mwd@gs-server-v-129:~/.ssh/id_rsa.pub ~/.ssh/129_rsa.pub
cat ~/.ssh/129_rsa.pub >> ~/.ssh/authorized_keys
在127机器操作
scp mwd@gs-server-v-128:~/.ssh/id_rsa.pub ~/.ssh/128_rsa.pub
cat ~/.ssh/128_rsa.pub >> ~/.ssh/authorized_keys
scp mwd@gs-server-v-127:~/.ssh/id_rsa.pub ~/.ssh/127_rsa.pub
cat ~/.ssh/127_rsa.pub >> ~/.ssh/authorized_keys
scp mwd@gs-server-v-129:~/.ssh/id_rsa.pub ~/.ssh/129_rsa.pub
cat ~/.ssh/129_rsa.pub >> ~/.ssh/authorized_keys
在129机器操作
scp mwd@gs-server-v-128:~/.ssh/id_rsa.pub ~/.ssh/128_rsa.pub
cat ~/.ssh/128_rsa.pub >> ~/.ssh/authorized_keys
scp mwd@gs-server-v-127:~/.ssh/id_rsa.pub ~/.ssh/127_rsa.pub
cat ~/.ssh/127_rsa.pub >> ~/.ssh/authorized_keys
scp mwd@gs-server-v-129:~/.ssh/id_rsa.pub ~/.ssh/129_rsa.pub
cat ~/.ssh/129_rsa.pub >> ~/.ssh/authorized_keys

下载zk
wget http://mirrors.hust.edu.cn/apache/zookeeper/zookeeper-3.4.6/zookeeper-3.4.6.tar.gz
wget http://mirrors.hust.edu.cn/apache/hadoop/common/hadoop-2.3.0/hadoop-2.3.0.tar.gz

配置环境变量

vi ~/.bash_profile
export ZK_HOME = /home/mwd/hadoop-2.3/zookeeper/zookeeper-3.4.6
export HADOOP_HOME=/home/mwd/hadoop-2.3/hadoop/hadoop-2.3.0
export PATH=$HADOOP_HOME/bin:$PATH
export PATH=$ZK_HOME/bin:$PATH
创建目录
mkdir -p $ZK_HOME/var/data
mkdir -p $ZK_HOME/var/datalog

修改配置文件
cp $ZK_HOME/conf/zoo_sample.cfg $ZK_HOME/conf/zoo.cfg
vi $ZK_HOME/conf/zoo.cfg
dataDir=/home/mwd/hadoop-2.3/zookeeper/zookeeper-3.4.6/var/data
dataLogDir=/home/mwd/hadoop-2.3/zookeeper/zookeeper-3.4.6/var/datalog

server.1=gs-server-v-127:2888:3888
server.2=gs-server-v-128:2888:3888

echo "1" > /home/mwd/hadoop-2.3/zookeeper/zookeeper-3.4.6/var/data/myid

$ZK_HOME/bin/zkServer.sh start
$ZK_HOME/bin/zkServer.sh stop

配置hadoop
修改env.sh
export JAVA_HOME=/opt/jdk1.7.0_51

修改core-site.xml文件

<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
<xi:include href="cmt.xml"/>


<property>
<name>fs.defaultFS</name>
<value>viewfs://cmt</value>
</property>


<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home/mwd/hadoop-2.3.0/hadoop/hadoop-2.3.0/journal</value>
</property>


<property>
  <name>hadoop.tmp.dir</name>
  <value>/home/mwd/hadoop-2.3.0/hadoop/hadoop-2.3.0/tmp</value>
 
</property>


<property>
<name>hadoop.proxyuser.root.hosts</name>
<value>*</value>
</property>
<property>
<name>hadoop.proxyuser.root.groups</name>
<value>*</value>
</property>





</configuration>

修改hdfs-site.xml

<configuration>
   <property>

      <name>dfs.replication</name>

      <value>2</value>

   </property>

<property>  
    <name>dfs.permissions</name>  
    <value>false</value>  
</property> 


<property>
<name>dfs.nameservices</name>
<value>ns1</value>
<description>Logical name for this new nameservice</description>
</property>


<property>
<name>dfs.ha.namenodes.ns1</name>
<value>nn1,nn2</value>
<description>Unique identifiers for each NameNode in the nameservice
</description>
</property>


<property>
    <name>dfs.namenode.rpc-address.ns1.nn1</name>
    <value>gs-server-v-127:8640</value>
</property>


<property>
    <name>dfs.namenode.http-address.ns1.nn1</name>
    <value>gs-server-v-127:50070</value>
</property>


<property>
    <name>dfs.namenode.rpc-address.ns1.nn2</name>
    <value>gs-server-v-128:8640</value>
</property>

<property>
    <name>dfs.namenode.http-address.ns1.nn2</name>
    <value>gs-server-v-128:50070</value>
</property>

<property>  
  <name>dfs.namenode.servicerpc-address.ns1.n1</name>  
  <value>gs-server-v-127:53310</value>  
</property>  
<property>  
  <name>dfs.namenode.servicerpc-address.ns1.n2</name>  
  <value>gs-server-v-128:53310</value>  
</property>



<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://gs-server-v-127:8485;gs-server-v-128:8485;gs-server-v-129:8485/ns1</value>
</property>

<property>
    <name>dfs.ha.automatic-failover.enabled</name>
    <value>true</value>
   
</property>



<property>
<name>dfs.client.failover.proxy.provider.ns1</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>



<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/mwd/.ssh/id_rsa</value>
</property>





<property>
    <name>ha.zookeeper.quorum</name>
    <value>gs-server-v-127:2181,gs-server-v-128:2181,gs-server-v-129:2128</value>
 
</property>

<property>
    <name>ha.zookeeper.session-timeout.ms</name>
    <value>5000</value>
    
</property>



 




</configuration>

修改yarn-site.xml

<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
<value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
<value>gs-server-v-127:8032</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
<value>gs-server-v-127:8030</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
<value>gs-server-v-127:8031</value>
</property>
<property>
<name>yarn.resourcemanager.admin.address</name>
<value>gs-server-v-127:8033</value>
</property>
<property>
<name>yarn.resourcemanager.webapp.address</name>
<value>gs-server-v-127:8088</value>
</property>
</configuration>

配置cmt.xml

<configuration>

    <property>
        <name>fs.viewfs.mounttable.cmt.link./user</name>
        <value>hdfs://ns1/user</value>
    </property>
    <property>
        <name>fs.viewfs.mounttable.cmt.link./fee</name>
        <value>hdfs://ns1/fee</value>
    </property>
</configuration>

maper-site.xml

<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.jobhistory.address</name>
<value>gs-server-v-127:10020</value>
</property>
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>gs-server-v-127:19888</value>
</property>
</configuration>

zkServer.sh start
zkServer.sh stop
===============启动 Hadoop======================

sudo netstat –apn | grep 9000

===============================================================分割线
$ZK_HOME_HA/bin/zkServer.sh start
$HADOOP_HOME/bin/hdfs zkfc -formatZK（首次需要）

$HADOOP_HOME/sbin/hadoop-daemon.sh start journalnode （all）

$HADOOP_HOME/sbin/hadoop-daemon.sh stop journalnode （all）

$HADOOP_HOME/bin/hdfs namenode -format （active）
$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode （active）

$HADOOP_HOME/bin/hdfs namenode -bootstrapStandby （standby）
$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode （standby）

$HADOOP_HOME/sbin/hadoop-daemon.sh start zkfc （both）

$HADOOP_HOME/sbin/hadoop-daemons.sh start datanode （active）
$HADOOP_HOME/sbin/start-yarn.sh （active）

查看进程：

ps -ef | grep java

检查端口

sudo netstat -apn |grep 9000

$HADOOP_HOME/bin/hdfs namenode -initializeSharedEdits