Centos6.5中编译hadoop2.x 并安装 运行wordCount

下载安装包：
http://mirror.bit.edu.cn/apache/maven/maven-3/3.0.5/binaries/
解压：
tar -zxvf apache-maven-3.0.5-bin.tar.gz
设置环境变量

export MAVEN_HOME=/app/lib/apache-maven-3.0.5
export PATH=$PATH:$MAVEN_HOME/bin

安装svn
yum install svn
安装libtool cmake
yum install autoconf automake libtool cmake
安装ncurses-devel
yum install ncurses-devel
安装openssl-devel
yum install openssl-devel
安装gcc
yum install gcc*
安装并设置protobuf
下载 http://pan.baidu.com/s/1pJlZubT

./configure
make
make check
make install

//验证
protoc

下载hadoop源包
svn checkout http://svn.apache.org/repos/asf/hadoop/common/tags/release-2.2.0
编译包
mvn package -Pdist,native -DskipTests -Dtar
file ./libhadoop.so.1.0.0
验证是否为64位

安装hadoop 2.x

解压安装包

mkdir tmp 
mkdir hdfs
mkdir hdfs/name
mkdir hdfs/data

配置hadoop-env.sh

	export HADOOP_CONF_DIR=/app/hadoop-2.2.0/etc/hadoop
	export HADOOP_HOME=/app/hadoop-2.2.0
	export PATH=$PATH:$HADOOP_HOME/bin
	export JAVA_HOME=/app/lib/jdk-8

配置/etc/profile

	export HADOOP_HOME=/app/hadoop-2.2.0
	export PATH=$PATH:$HADOOP_HOME/bin

配置yarn-env.sh

	export JAVA_HOME=/app/lib/jdk-8

配置core-site.xml

<configuration>
   <property>
    <name>fs.default.name</name>
     <value>hdfs://hadoop:9000</value>
   </property>
   <property>
     <name>fs.defaultFS</name>
   </property>
   <property>
     <name>io.file.buffer.size</name>
     <value>131072</value>
   </property>
   <property>
     <name>hadoop.tmp.dir</name>
     <value>file:/app/hadoop-2.2.0/tmp</value>
     <description>A base for other temporary directories.</description>
  </property>
   <property>
     <name>hadoop.proxyuser.hduser.hosts</name>
     <value>*</value>
   </property>
   <property>
     <name>hadoop.proxyuser.hduser.groups</name>
     <value>*</value>
   </property>
 </configuration>

配置hdfs-site.xml

	<configuration>
   <property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>hadoop:9001</value>
   </property>
   <property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/app/hadoop-2.2.0/hdfs/name</value>
   </property>
   <property>
    <name>dfs.datanode.data.dir</name>
    <value>file:/app/hadoop-2.2.0/hdfs/data</value>
   </property>
   <property>
    <name>dfs.replication</name>
    <value>1</value>
   </property>
   <property>
    <name>dfs.webhdfs.enabled</name>
    <value>true</value>
   </property>
 </configuration>

配置mapred-site.xml

<configuration>
   <property>
     <name>mapreduce.framework.name</name>
     <value>yarn</value>
   </property>
   <property>
     <name>mapreduce.jobhistory.address</name>
     <value>hadoop:10020</value>
   </property>
   <property>
     <name>mapreduce.jobhistory.webapp.address</name>
     <value>hadoop:19888</value>
   </property>
 </configuration>

配置yarn-site.xml

 <configuration>
   <property>
     <name>yarn.nodemanager.aux-services</name>
     <value>mapreduce_shuffle</value>
   </property>
   <property>
     <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
     <value>org.apache.hadoop.mapred.ShuffleHandler</value>
   </property>
   <property>
     <name>yarn.resourcemanager.address</name>
     <value>hadoop:8032</value>
   </property>
   <property>
     <name>yarn.resourcemanager.scheduler.address</name>
     <value>hadoop:8030</value>
   </property>
   <property>
     <name>yarn.resourcemanager.resource-tracker.address</name>
     <value>hadoop:8031</value>
   </property>
   <property>
     <name>yarn.resourcemanager.admin.address</name>
     <value>hadoop:8033</value>
   </property>
   <property>
     <name>yarn.resourcemanager.webapp.address</name>
     <value>hadoop:8088</value>
   </property>
 </configuration>

设置slaves

vim slaves
hadoop

	./hadoop namenode -format
	./start-dfs.sh
	./start-yarn.sh

	jps:
	  namenode
	  datanode
	  secondarynamenode
	  resourcemanager
	  nodemanager

测试wordCount

hdfs中创建目录
hadoop fs -mkdir -p /class3/input
准备测试数据
hadoop fs -copyFromLocal …/etc/hadoop/* /class3/input
运行wordCount
hadoop jar …/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.2.0.jar wordcount /class3/input /class3/output
查看运行结果
hadoop fs -cat /class3/output/part-r-00000 | less