hadoop2.5.2eclipse插件编译

最近在搞hadoop，所以编译hadoop-eclipse插件是少不了的 ！

先说下我的编译环境：

hadoop: hadoop-2.5.2

OS：ubuntu 14.4 64位（虚拟机）

eclipse: eclipse-jee-luna-SR1-linux-gtk-x86_64.tar.gz

1.下载源码：

https://github.com/winghc/hadoop2x-eclipse-plugin

2.修改配置文件

libraries.properties

#   Licensed under the Apache License, Version 2.0 (the"License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copyof the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless requiredby applicable lawor agreed toin writing, software
#   distributed under the Licenseis distributed on an "AS IS"BASIS,
#   WITHOUT WARRANTIESOR CONDITIONS OF ANY KIND, either express or implied.
#   See the Licensefor the specific language governing permissionsand
#   limitations under the License.
 
#This properties file lists the versionsof the various artifacts usedby hadoop and components.
#It drives ivy and the generation of a maven POM
# This isthe version of hadoop we are generating
<span style="background-color: rgb(255, 0, 0);">hadoop.version=2.2.0</span>
hadoop-gpl-compression.version=0.1.0
 
#These are the versionsof our dependencies (inalphabetical order)
apacheant.version=1.7.0
ant-task.version=2.0.10
 
asm.version=3.2
aspectj.version=1.6.5
aspectj.version=1.6.11
 
checkstyle.version=4.2
 
commons-cli.version=1.2
commons-codec.version=1.4
=commons-collections.version=3.1
commons-configuration.version=1.6
commons-daemon.version=1.0.13
=commons-httpclient.version=3.0.1
=commons-lang.version=2.4
=commons-logging.version=1.0.4
commons-logging-api.version=1.0.4
=commons-math.version=2.1
commons-el.version=1.0
commons-fileupload.version=1.2
=commons-io.version=2.1
commons-net.version=3.1
core.version=3.1.1
coreplugin.version=1.3.2
 
hsqldb.version=1.8.0.10
 
ivy.version=2.1.0
 
jasper.version=5.5.12
=jackson.version=1.8.8
#notable to figureout the version of jsp & jsp-api version to get it resolved throught ivy
# but still declared hereas we are going to have a localcopy from the lib folder
jsp.version=2.1
jsp-api.version=5.5.12
jsp-api-2.1.version=6.1.14
jsp-2.1.version=6.1.14
jets3t.version=0.6.1
jetty.version=6.1.26
jetty-util.version=6.1.26
=jersey-core.version=1.8
=jersey-json.version=1.8
=jersey-server.version=1.8
=junit.version=4.5
jdeb.version=0.8
jdiff.version=1.0.9
json.version=1.0
 
kfs.version=0.1
 
log4j.version=1.2.15
lucene-core.version=2.3.1
 
mockito-all.version=1.8.5
jsch.version=0.1.42
 
oro.version=2.0.8
 
rats-lib.version=0.5.1
 
servlet.version=4.0.6
servlet-api.version=2.5
slf4j-api.version=1.4.3
slf4j-log4j12.version=1.4.3
 
wagon-http.version=1.0-beta-2
xmlenc.version=0.52
xerces.version=1.4.4

以上标"="的部分我修改为hadoop2.5.2所用的jar包版本

 

hadoop.version=2.5.2
commons-collections.version=3.2.1
commons-httpclient.version=3.1
commons-lang.version=2.6
commons-logging.version=1.1.3
commons-math.version=3.1.1
commons-io.version=2.4
jackson.version=1.9.13
jersey-core.version=1.9
jersey-json.version=1.9
jersey-server.version=1.9
junit.version=4.11

 

build.xml

<?xml version="1.0"encoding="UTF-8" standalone="no"?>
 
<!--
   Licensedto the Apache Software Foundation (ASF) under oneor more
   contributor license agreements.  See the NOTICE file distributedwith
   thiswork foradditional information regarding copyright ownership.
   The ASF licenses this fileto You under the Apache License, Version 2.0
   (the"License"); you maynot use this file except in compliance with
   the License.  You may obtain a copyof the License at
 
       http://www.apache.org/licenses/LICENSE-2.0
 
   Unless requiredby applicable lawor agreed toin writing, software
   distributed under the Licenseis distributed on an "AS IS"BASIS,
   WITHOUT WARRANTIESOR CONDITIONS OF ANY KIND, either express or implied.
   See the Licensefor the specific language governing permissionsand
   limitations under the License.
-->
 
<project default="jar"name="eclipse-plugin">
 
  <import file="../build-contrib.xml"/>
 
  <path id="eclipse-sdk-jars">
    <fileset dir="${eclipse.home}/plugins/">
      <includename="org.eclipse.ui*.jar"/>
      <includename="org.eclipse.jdt*.jar"/>
      <includename="org.eclipse.core*.jar"/>
      <includename="org.eclipse.equinox*.jar"/>
      <includename="org.eclipse.debug*.jar"/>
      <includename="org.eclipse.osgi*.jar"/>
      <includename="org.eclipse.swt*.jar"/>
      <includename="org.eclipse.jface*.jar"/>
 
      <includename="org.eclipse.team.cvs.ssh2*.jar"/>
      <includename="com.jcraft.jsch*.jar"/>
    </fileset>
  </path>
 
  <path id="hadoop-sdk-jars">
    <fileset dir="${hadoop.home}/share/hadoop/mapreduce">
      <includename="hadoop*.jar"/>
    </fileset>
    <fileset dir="${hadoop.home}/share/hadoop/hdfs">
      <includename="hadoop*.jar"/>
    </fileset>
    <fileset dir="${hadoop.home}/share/hadoop/common">
      <includename="hadoop*.jar"/>
    </fileset>
  </path>
 
 
 
  <!-- Override classpath to include Eclipse SDK jars -->
  <path id="classpath">
    <pathelement location="${build.classes}"/>
    <!--pathelement location="${hadoop.root}/build/classes"/-->
    <path refid="eclipse-sdk-jars"/>
    <path refid="hadoop-sdk-jars"/>
  </path>
 
  <!-- Skip building if eclipse.home is unset. -->
  <targetname="check-contrib"unless="eclipse.home">
    <propertyname="skip.contrib"value="yes"/>
    <echo message="eclipse.home unset: skipping eclipse plugin"/>
  </target>
 
 <targetname="compile"depends="init, ivy-retrieve-common"unless="skip.contrib">
    <echo message="contrib: ${name}"/>
    <javac
     encoding="${build.encoding}"
     srcdir="${src.dir}"
     includes="**/*.java"
     destdir="${build.classes}"
     debug="${javac.debug}"
     deprecation="${javac.deprecation}">
     <classpath refid="classpath"/>
    </javac>
  </target>
 
  <!-- Override jar target to specify manifest -->
  <targetname="jar"depends="compile" unless="skip.contrib">
    <mkdir dir="${build.dir}/lib"/>
    <copy  todir="${build.dir}/lib/"verbose="true">
          <fileset dir="${hadoop.home}/share/hadoop/mapreduce">
           <includename="hadoop*.jar"/>
          </fileset>
    </copy>
    <copy  todir="${build.dir}/lib/"verbose="true">
          <fileset dir="${hadoop.home}/share/hadoop/common">
           <includename="hadoop*.jar"/>
          </fileset>
    </copy>
    <copy  todir="${build.dir}/lib/"verbose="true">
          <fileset dir="${hadoop.home}/share/hadoop/hdfs">
           <includename="hadoop*.jar"/>
          </fileset>
    </copy>
    <copy  todir="${build.dir}/lib/"verbose="true">
          <fileset dir="${hadoop.home}/share/hadoop/yarn">
           <includename="hadoop*.jar"/>
          </fileset>
    </copy>
 
    <copy  todir="${build.dir}/classes"verbose="true">
          <fileset dir="${root}/src/java">
           <includename="*.xml"/>
          </fileset>
    </copy>
 
 
 
    <copy file="${hadoop.home}/share/hadoop/common/lib/protobuf-java-${protobuf.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/log4j-${log4j.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/commons-cli-${commons-cli.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/commons-configuration-${commons-configuration.version}.jar" todir="${build.dir}/lib"verbose="true"/>
 +   <span style="color:#33CC00;"><copy file="${hadoop.home}/share/hadoop/common/lib/commons-collections-${commons-collections.version}.jar"todir="${build.dir}/lib"verbose="true"/></span>
    <copy file="${hadoop.home}/share/hadoop/common/lib/commons-lang-${commons-lang.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/jackson-core-asl-${jackson.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/jackson-mapper-asl-${jackson.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/slf4j-log4j12-${slf4j-log4j12.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/slf4j-api-${slf4j-api.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/guava-${guava.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/hadoop-auth-${hadoop.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/commons-cli-${commons-cli.version}.jar" todir="${build.dir}/lib"verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/netty-${netty.version}.jar" todir="${build.dir}/lib"verbose="true"/>
 
    <jar
      jarfile="${build.dir}/hadoop-${name}-${version}.jar"
      manifest="${root}/META-INF/MANIFEST.MF">
      <manifest>
     <attributename="Bundle-ClassPath"
        value="classes/,
 lib/hadoop-mapreduce-client-core-${hadoop.version}.jar,
 lib/hadoop-mapreduce-client-common-${hadoop.version}.jar,
 lib/hadoop-mapreduce-client-jobclient-${hadoop.version}.jar,
 lib/hadoop-auth-${hadoop.version}.jar,
 lib/hadoop-common-${hadoop.version}.jar,
 lib/hadoop-hdfs-${hadoop.version}.jar,
 lib/protobuf-java-${protobuf.version}.jar,
 lib/log4j-${log4j.version}.jar,
 + lib/commons-collections-${commons-collections.version}.jar,
 =lib/commons-cli-1.2.jar,
 =lib/commons-configuration-1.6.jar,
 =lib/commons-httpclient-3.1.jar,
 =lib/commons-lang-2.5.jar,
 =lib/jackson-core-asl-1.8.8.jar,
 =lib/jackson-mapper-asl-1.8.8.jar,
 =lib/slf4j-log4j12-1.7.5.jar,
 =lib/slf4j-api-1.7.5.jar,
 lib/guava-${guava.version}.jar,
 lib/netty-${netty.version}.jar"/>
     </manifest>
      <fileset dir="${build.dir}"includes="classes/ lib/"/>
      <!--fileset dir="${build.dir}" includes="*.xml"/-->
      <fileset dir="${root}"includes="resources/ plugin.xml"/>
    </jar>
  </target>
 
</project>

以上以标有“=”的部分我修改为

 

lib/commons-cli-${commons-cli.version}.jar,
lib/commons-configuration-${commons-configuration.version}.jar,
lib/commons-httpclient-${commons-httpclient.version}.jar,
lib/commons-lang-${commons-lang.version}.jar,
lib/jackson-core-asl-${jackson.version},
lib/jackson-mapper-asl-${jackson.version}.jar,
lib/slf4j-log4j12-${slf4j-log4j12.version}.jar,
lib/slf4j-api-${slf4j-api.version}.jar,

标有“+”的部分是我加上的。

 

makePlus.sh

原文件内容

 

ant jar -Dversion=2.0.4 -Declipse.home=/opt/eclipse -Dhadoop.home=/usr/share/hadoop

修改后

 

 

nt jar -Dversion=2.5.2 -Declipse.home=/home/hadoop/eclipse -Dhadoop.home=/home/hadoop/hadoop-2.5.2

使之可以执行

 

 

chmod +X makePlus.sh

 

3.编译

进入makePlus.sh所在目录，执行我们修改好的makePlus.sh文件 ./makePlus.sh

在这过程中有一步ivy-resolve-common很长时间没反应，我不清楚是死掉了还是在运行中所以就一直等啊等，上网查说这一步是很花时间的我也就没有在意，

可是都一下午了还是没有任何提示成功或者失败。无奈，拿起qq搜了几个hadoop的群，进去问下，有人说慢很正常，网络卡顿了吧。我这一想有可能是网络不通，可是我的虚拟机是能联着网的，只有一种可能了，就是编译过程中需求去外国的网站下载所依赖的包，需要翻墙。想到这里我立马开个翻墙软件，再次执行 ./makePlus.sh，奇迹出现了！哈哈 编译成功了

 

Buildfile: /home/hadoop/hadoop2x-eclipse-plugin/src/contrib/eclipse-plugin/build.xml
 
check-contrib:
 
init:
     [echo] contrib: eclipse-plugin
 
init-contrib:
 
ivy-download:
      [get] Getting:http://repo2.maven.org/maven2/org/apache/ivy/ivy/2.1.0/ivy-2.1.0.jar
      [get]To: /home/hadoop/hadoop2x-eclipse-plugin/ivy/ivy-2.1.0.jar
      [get]Not modified - so not downloaded
 
ivy-probe-antlib:
 
ivy-init-antlib:
 
ivy-init:
[ivy:configure] :: Ivy 2.1.0 - 20090925235825 ::http://ant.apache.org/ivy/ ::
[ivy:configure] :: loading settings :: file = /home/hadoop/hadoop2x-eclipse-plugin/ivy/ivysettings.xml
 
ivy-resolve-common:
     [echo] loglevel:quiet;ant.project.name:eclipse-plugin
 
ivy-retrieve-common:
[ivy:cachepath] DEPRECATED:'ivy.conf.file' is deprecated, use 'ivy.settings.file'instead
[ivy:cachepath] :: loading settings :: file = /home/hadoop/hadoop2x-eclipse-plugin/ivy/ivysettings.xml
 
compile:
     [echo] contrib: eclipse-plugin
    [javac] /home/hadoop/hadoop2x-eclipse-plugin/src/contrib/eclipse-plugin/build.xml:76: warning:'includeantruntime' was not set, defaultingto build.sysclasspath=last;set to false for repeatablebuilds
 
jar:
     [copy] Copying 1 fileto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/jackson-core-asl-1.9.13.jarto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/jackson-core-asl-1.9.13.jar
     [copy] Copying 1 fileto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/jackson-mapper-asl-1.9.13.jarto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/jackson-mapper-asl-1.9.13.jar
     [copy] Copying 1 fileto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jarto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/slf4j-log4j12-1.7.5.jar
     [copy] Copying 1 fileto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/slf4j-api-1.7.5.jarto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/slf4j-api-1.7.5.jar
     [copy] Copying 1 fileto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/guava-11.0.2.jarto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/guava-11.0.2.jar
     [copy] Copying 1 fileto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/hadoop-auth-2.5.2.jarto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/hadoop-auth-2.5.2.jar
     [copy] Copying 1 fileto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/netty-3.6.2.Final.jarto /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/netty-3.6.2.Final.jar
   <span style="font-size:18px;"></span>  [jar] Building jar: /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/hadoop-eclipse-plugin-2.5.2.jar
 
BUILD SUCCESSFUL
Total time: 16 seconds
 
log4j:WARN No appenders could be foundfor logger (org.apache.hadoop.util.Shell).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig formore info.

[jar] Building jar: /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/hadoop-eclipse-plugin-2.5.2.jar就是编译好的jar包所在路径了！

 

4.测试

把编译好的hadoop-eclipse-plugin-2.5.2.jar放在eclipse的plugins目录下，重启eclipse，配置好后发现不能 new hadoop location,现象是点击后没有任何反应。只好退出eclipse 在终端里进入eclipse 目录执行

 

./eclipse -clean -consolelog -debug

重新进入eclipse 点击 new hadoop location,查看终端里报出的异常信息java.lang.NoClassDefFoundError

 

经过一番百度之后build.xml添加绿色部分重新编译，放在eclipse的plugins目录下重启eclipse，便可以使用了

=