1 Kylin install-hadoop

系统环境

OS:10.12.5 (16F73)
java version “1.8.0_121”
Hadoop 2.7.5

MAC ssh localhost设置

---

1. 偏好设置－共享－打开远程共享
2. ssh-keygen -t dsa -P ‘’ -f ~/.ssh/id_dsa
3. cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

hadoop安装

---

1. 下载
   http://hadoop.apache.org/docs/r2.7.5/
2. 官方安装手册
   http://hadoop.apache.org/docs/r2.7.5/hadoop-project-dist/hadoop-common/SingleCluster.html
   http://hadoop.apache.org/docs/r2.7.5/hadoop-project-dist/hadoop-common/ClusterSetup.html
3. 配置

• hadoop-2.7.5/etc/hadoop/yarn-site.xml

<property>
    <name>yarn.nodemanager.local-dirs</name>
    <value>${hadoop.tmp.dir}/nmlocaldir</value>
</property>
<property>
    <name>yarn.resourcemanager.hostname</name>
      <value>localhost</value>
</property>
<property>
      <name>yarn.nodemanager.aux-services</name>
      <value>mapreduce_shuffle</value>
</property>
<property>
      <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
      <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
 <property>
    <name>yarn.application.classpath</name>
    <value>
        /home/soft/hadoop-2.7.5/etc/hadoop,
        /home/soft/hadoop-2.7.5/share/hadoop/common/*,
        /home/soft/hadoop-2.7.5/share/hadoop/common/lib/*,
        /home/soft/hadoop-2.7.5/share/hadoop/hdfs/*,
        /home/soft/hadoop-2.7.5/share/hadoop/hdfs/lib/*,
        /home/soft/hadoop-2.7.5/share/hadoop/mapreduce/*,
        /home/soft/hadoop-2.7.5/share/hadoop/mapreduce/lib/*,
        /home/soft/hadoop-2.7.5/share/hadoop/yarn/*,
        /home/soft/hadoop-2.7.5/share/hadoop/yarn/lib/*
    </value>
</property>

• mapred-site.xml

<property>  
   <name>mapred.job.tracker</name>  
   <value>localhost:9001</value>  
   <description>jobtracker标识:端口号，不是URI</description> 
 </property>  
 <property>   
   <name>mapred.local.dir</name>   
   <value>/tmp/local</value>   
   <description>tasktracker上执行mapreduce程序时的本地目录</description>   
 </property>
 <property>
   <name>mapreduce.framework.name</name>
   <value>yarn</value>
 </property>
 <property>
   <name>mapred.child.tmp</name> 
   <value>/tmp</value>
 </property>
 <property>
   <name>mapreduce.jobhistory.address</name>
   <value>localhost:10020</value>
 </property>
 <property>
   <name>mapreduce.jobhistory.webapp.address</name>
   <value>localhost:19888</value>
 </property>
 <property>
   <name>mapreduce.application.classpath</name>
   <value>
       /home/soft/hadoop-2.7.5/etc/hadoop,
       /home/soft/hadoop-2.7.5/share/hadoop/common/*,
       /home/soft/hadoop-2.7.5/share/hadoop/common/lib/*,
       /home/soft/hadoop-2.7.5/share/hadoop/hdfs/*,
       /home/soft/hadoop-2.7.5/share/hadoop/hdfs/lib/*,
       /home/soft/hadoop-2.7.5/share/hadoop/mapreduce/*,
       /home/soft/hadoop-2.7.5/share/hadoop/mapreduce/lib/*,
       /home/soft/hadoop-2.7.5/share/hadoop/yarn/*,
       /home/soft/hadoop-2.7.5/share/hadoop/yarn/lib/*
   </value>
 </property>

• core-site.xml

<property>
  <name>fs.defaultFS</name>
  <value>hdfs://localhost:8001</value>
</property>
<property>
  <name>hadoop.tmp.dir</name>
  <value>/tmp/hadooptmp</value>
</property>

• hdfs-site.xml

  <property>
     <name>dfs.replication</name>
     <value>1</value>
  </property>
  <property>
     <name>dfs.namenode.http-address</name>
     <value>localhost:50070</value>
  </property>
  <property>
     <name>dfs.namenode.secondary.http-address</name>
     <value>localhost:50090</value>
  </property>
  

4、增加环境变量 （在~/.bash_profile中）
export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_121.jdk/Contents/Home

export HADOOP_HOME=/Users/soft/hadoop-2.7.5
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
export HADOOP_CONF=/Users/soft/hadoop-2.7.5/etc/hadoop
export HADOOP_LIB=${HADOOP_HOME}/share/hadoop/mapreduce/lib:${HADOOP_HOME}/share/hadoop/

export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin:${HADOOP_LIB}

5、启动&测试
$HADOOP_HOME/bin/hadoop namenode -format
$HADOOP_HOME/sbin/start-all.sh

6、校验
sbin $ jps
1504 NodeManager
1153 DataNode
1539 Jps
1060 NameNode
1271 SecondaryNameNode
1404 ResourceManager

参考资料

http://blog.csdn.net/woshicsdn7547/article/details/51934735