hadoop2.0.5配置

最新推荐文章于 2018-12-03 18:48:00 发布

anghlq

最新推荐文章于 2018-12-03 18:48:00 发布

阅读量1.8k

点赞数

分类专栏：云在天涯

本文链接：https://blog.csdn.net/anghlq/article/details/10187051

版权

云在天涯专栏收录该内容

77 篇文章 0 订阅

订阅专栏

修改日志的Logger Level或者appender，需要修改 sbin/hadoop-daemon.sh

export HADOOP_ROOT_LOGGER=${HADOOP_ROOT_LOGGER:-"INFO,DRFA"}
export HADOOP_SECURITY_LOGGER=${HADOOP_SECURITY_LOGGER:-"INFO,RFAS"}
export HDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-"INFO,NullAppender"}

配置zookeeper，NN的HA依赖它

配置

vi /opt/zookeeper/conf/zoo.cfg

dataDir=/home1/hadoop/zookeeper/data
dataLogDir=/home1/hadoop/zookeeper/zk_log
logDir=/home1/hadoop/zookeeper/logs

server.110=Server110:2888:3888
server.111=Server111:2888:3888
server.112=Server112:2888:3888

在所有zk机器上创建目录和myid文件

mkdir -p /home1/hadoop/zookeeper/data
mkdir -p /home1/hadoop/zookeeper/zk_log
mkdir -p /home1/hadoop/zookeeper/logs
ifconfig|grep 192|awk -F'.' '{print $4}'|awk '{print $1}' >/home1/hadoop/zookeeper/data/myid

在所有zk机器上启动

/opt/zookeeper/bin/zkServer.sh start

配置Hadoop

1. 配置环境变量

vi /etc/profile

vi /opt/hadoop/etc/hadoop/hadoop-env.sh
export JAVA_HOME=/usr/java/jdk1.7.0_25
export HADOOP_PREFIX="/opt/hadoop"
export PATH=$PATH:$HADOOP_PREFIX/bin
export PATH=$PATH:$HADOOP_PREFIX/sbin
export HADOOP_MAPRED_HOME=${HADOOP_PREFIX}
export HADOOP_COMMON_HOME=${HADOOP_PREFIX}
export HADOOP_HDFS_HOME=${HADOOP_PREFIX}
export YARN_HOME=${HADOOP_PREFIX}
export HADOOP_SSH_OPTS="-p 9922"
export HADOOP_CONF_DIR=${HADOOP_PREFIX}/etc/hadoop
export HDFS_CONF_DIR=${HADOOP_PREFIX}/etc/hadoop

export YARN_CONF_DIR=${HADOOP_PREFIX}/etc/hadoop

2. 配置 /opt/hadoop/etc/hadoop

配置core-site.xml文件
<property>
   <name>hadoop.tmp.dir</name>
   <value>/home1/hadoop/tmp</value>
</property>
<property>
   <name>fs.default.name</name>
   <value>hdfs://ns1</value>
</property>

---创建并配置slaves：vi slaves 并添加以下内容
Server110
Server111
Server112
Server113
Server114

---配置hdfs-site.xml

<property>
<name>dfs.datanode.max.transfer.threads</name>
<value>8196</value>
</property>
<property>
<name>dfs.replication</name>
<value>2</value>
</property>
<property>
<name>dfs.nameservices</name>
<value>ns1</value>
</property>
<property>
<name>dfs.ha.namenodes.ns1</name>
<value>ns1n1,ns1n2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.ns1.ns1n1</name>
<value>Server110:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.ns1.ns1n2</name>
<value>Server113:8020</value>
</property>
<property>
<name>dfs.namenode.http-address.ns1.ns1n1</name>
<value>Server110:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.ns1.ns1n2</name>
<value>Server113:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://Server110:8485;Server111:8485;Server112:8485/ns1</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.ns1</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence(huanglq:9922)</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/home/hadoop/.ssh/id_rsa</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>30000</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/home1/hadoop/journal/node/local/data</value>
</property>
<property>
   <name>dfs.ha.automatic-failover.enabled</name>
   <value>true</value>
</property>
<property>
   <name>ha.zookeeper.quorum</name>
   <value>Server110:2181,Server111:2181,Server112:2181</value>
</property>

<property>
   <name>dfs.namenode.name.dir</name>
   <value>/home1/hadoop/hdfs/name</value>
   <final>true</final>
</property>

<property>
   <name>dfs.federation.nameservice.id</name>
   <value>ns1</value>
</property>

<property>
   <name>dfs.federation.nameservices</name>
   <value>ns1</value>
   
</property>

<property>
<name>dfs.datanode.du.reserved</name>
<value>1073741824</value>
<description>Reserved space in bytes per volume. Always leave this much space free for non dfs use.
</description>

</property>

<property>
<name>dfs.datanode.data.dir</name>

<value>/home1/hadoop/hdfs/data</value>

<final>true</final>
</property>

</configuration>

----配置yarn-site.xml
<configuration>


<property>
   <name>yarn.resourcemanager.address</name>
   <value>Server110:18040</value>
</property>

<property>
   <name>yarn.resourcemanager.scheduler.address</name>
   <value>Server110:18030</value>
</property>

<property>
   <name>yarn.resourcemanager.webapp.address</name>
   <value>Server110:18088</value>
</property>

<property>
   <name>yarn.resourcemanager.resource-tracker.address</name>
   <value>Server110:18025</value>
</property>

<property>
   <name>yarn.resourcemanager.admin.address</name>
   <value>Server110:18141</value>
</property>

<property>
   <name>yarn.nodemanager.aux-services</name>
   <value>mapreduce.shuffle</value>
</property>
</configuration>

3. 初始化namenode

hdfs zkfc -formatZK

在所有的nn机器上执行

/opt/hadoop/bin/hdfs namenode -format ns1

4. 启动
sbin/start-dfs.sh
sbin/start-yarn.sh