关闭

Hadoop之——hadoop2.5.2eclipse插件编译

标签: hadoopeclipseeclipse插件
786人阅读 评论(0) 收藏 举报
分类:

转载请注明出处:http://blog.csdn.net/l1028386804/article/details/51542414

最近在搞hadoop,所以编译hadoop-eclipse插件是少不了的 !
先说下我的编译环境:
hadoop: hadoop-2.5.2
OS:CentOS6.5 64位(VMWare虚拟机)
eclipse: eclipse-jee-luna-SR1-linux-gtk-x86_64.tar.gz

1、下载源码

https://github.com/winghc/hadoop2x-eclipse-plugin

2.修改配置文件

libraries.properties

我的路径为:/usr/local/hadoop/hadoop2x-eclipse-plugin-master/ivy/libraries.properties

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
 
#This properties file lists the versions of the various artifacts used by hadoop and components.
#It drives ivy and the generation of a maven POM
# This is the version of hadoop we are generating
<span style="background-color: rgb(255, 0, 0);">hadoop.version=2.2.0</span>
hadoop-gpl-compression.version=0.1.0
 
#These are the versions of our dependencies (in alphabetical order)
apacheant.version=1.7.0
ant-task.version=2.0.10
 
asm.version=3.2
aspectj.version=1.6.5
aspectj.version=1.6.11
 
checkstyle.version=4.2
 
commons-cli.version=1.2
commons-codec.version=1.4
=commons-collections.version=3.1
commons-configuration.version=1.6
commons-daemon.version=1.0.13
=commons-httpclient.version=3.0.1
=commons-lang.version=2.4
=commons-logging.version=1.0.4
commons-logging-api.version=1.0.4
=commons-math.version=2.1
commons-el.version=1.0
commons-fileupload.version=1.2
=commons-io.version=2.1
commons-net.version=3.1
core.version=3.1.1
coreplugin.version=1.3.2
 
hsqldb.version=1.8.0.10
 
ivy.version=2.1.0
 
jasper.version=5.5.12
=jackson.version=1.8.8
#not able to figureout the version of jsp & jsp-api version to get it resolved throught ivy
# but still declared here as we are going to have a local copy from the lib folder
jsp.version=2.1
jsp-api.version=5.5.12
jsp-api-2.1.version=6.1.14
jsp-2.1.version=6.1.14
jets3t.version=0.6.1
jetty.version=6.1.26
jetty-util.version=6.1.26
=jersey-core.version=1.8
=jersey-json.version=1.8
=jersey-server.version=1.8
=junit.version=4.5
jdeb.version=0.8
jdiff.version=1.0.9
json.version=1.0
 
kfs.version=0.1
 
log4j.version=1.2.15
lucene-core.version=2.3.1
 
mockito-all.version=1.8.5
jsch.version=0.1.42
 
oro.version=2.0.8
 
rats-lib.version=0.5.1
 
servlet.version=4.0.6
servlet-api.version=2.5
slf4j-api.version=1.4.3
slf4j-log4j12.version=1.4.3
 
wagon-http.version=1.0-beta-2
xmlenc.version=0.52
xerces.version=1.4.4
以上标"="的部分我修改为hadoop2.5.2所用的jar包版本
hadoop.version=2.5.2
commons-collections.version=3.2.1
commons-httpclient.version=3.1
commons-lang.version=2.6
commons-logging.version=1.1.3
commons-math.version=3.1.1
commons-io.version=2.4
jackson.version=1.9.13
jersey-core.version=1.9
jersey-json.version=1.9
jersey-server.version=1.9
junit.version=4.11

build.xml

我的路径为:usr/local/hadoop/hadoop2x-eclipse-plugin-master/src/contrib/eclipse-plugin/build.xml

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 
<!--
   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at
 
       http://www.apache.org/licenses/LICENSE-2.0
 
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
-->
 
<project default="jar" name="eclipse-plugin">
 
  <import file="../build-contrib.xml"/>
 
  <path id="eclipse-sdk-jars">
    <fileset dir="${eclipse.home}/plugins/">
      <include name="org.eclipse.ui*.jar"/>
      <include name="org.eclipse.jdt*.jar"/>
      <include name="org.eclipse.core*.jar"/>
      <include name="org.eclipse.equinox*.jar"/>
      <include name="org.eclipse.debug*.jar"/>
      <include name="org.eclipse.osgi*.jar"/>
      <include name="org.eclipse.swt*.jar"/>
      <include name="org.eclipse.jface*.jar"/>
 
      <include name="org.eclipse.team.cvs.ssh2*.jar"/>
      <include name="com.jcraft.jsch*.jar"/>
    </fileset> 
  </path>
 
  <path id="hadoop-sdk-jars">
    <fileset dir="${hadoop.home}/share/hadoop/mapreduce">
      <include name="hadoop*.jar"/>
    </fileset> 
    <fileset dir="${hadoop.home}/share/hadoop/hdfs">
      <include name="hadoop*.jar"/>
    </fileset> 
    <fileset dir="${hadoop.home}/share/hadoop/common">
      <include name="hadoop*.jar"/>
    </fileset> 
  </path>
 
 
 
  <!-- Override classpath to include Eclipse SDK jars -->
  <path id="classpath">
    <pathelement location="${build.classes}"/>
    <!--pathelement location="${hadoop.root}/build/classes"/-->
    <path refid="eclipse-sdk-jars"/>
    <path refid="hadoop-sdk-jars"/>
  </path>
 
  <!-- Skip building if eclipse.home is unset. -->
  <target name="check-contrib" unless="eclipse.home">
    <property name="skip.contrib" value="yes"/>
    <echo message="eclipse.home unset: skipping eclipse plugin"/>
  </target>
 
 <target name="compile" depends="init, ivy-retrieve-common" unless="skip.contrib">
    <echo message="contrib: ${name}"/>
    <javac
     encoding="${build.encoding}"
     srcdir="${src.dir}"
     includes="**/*.java"
     destdir="${build.classes}"
     debug="${javac.debug}"
     deprecation="${javac.deprecation}">
     <classpath refid="classpath"/>
    </javac>
  </target>
 
  <!-- Override jar target to specify manifest -->
  <target name="jar" depends="compile" unless="skip.contrib">
    <mkdir dir="${build.dir}/lib"/>
    <copy  todir="${build.dir}/lib/" verbose="true">
          <fileset dir="${hadoop.home}/share/hadoop/mapreduce">
           <include name="hadoop*.jar"/>
          </fileset>
    </copy>
    <copy  todir="${build.dir}/lib/" verbose="true">
          <fileset dir="${hadoop.home}/share/hadoop/common">
           <include name="hadoop*.jar"/>
          </fileset>
    </copy>
    <copy  todir="${build.dir}/lib/" verbose="true">
          <fileset dir="${hadoop.home}/share/hadoop/hdfs">
           <include name="hadoop*.jar"/>
          </fileset>
    </copy>
    <copy  todir="${build.dir}/lib/" verbose="true">
          <fileset dir="${hadoop.home}/share/hadoop/yarn">
           <include name="hadoop*.jar"/>
          </fileset>
    </copy>
 
    <copy  todir="${build.dir}/classes" verbose="true">
          <fileset dir="${root}/src/java">
           <include name="*.xml"/>
          </fileset>
    </copy>
 
 
 
    <copy file="${hadoop.home}/share/hadoop/common/lib/protobuf-java-${protobuf.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/log4j-${log4j.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/commons-cli-${commons-cli.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/commons-configuration-${commons-configuration.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <!--下面这行标注为绿色,文本插件的原因显示不出来,将样式显示了出来-->
 +   <span style="color:#33CC00;"><copy file="${hadoop.home}/share/hadoop/common/lib/commons-collections-${commons-collections.version}.jar" todir="${build.dir}/lib" verbose="true"/></span>
    <copy file="${hadoop.home}/share/hadoop/common/lib/commons-lang-${commons-lang.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/jackson-core-asl-${jackson.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/jackson-mapper-asl-${jackson.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/slf4j-log4j12-${slf4j-log4j12.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/slf4j-api-${slf4j-api.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/guava-${guava.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/hadoop-auth-${hadoop.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/commons-cli-${commons-cli.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
    <copy file="${hadoop.home}/share/hadoop/common/lib/netty-${netty.version}.jar"  todir="${build.dir}/lib" verbose="true"/>
 
    <jar
      jarfile="${build.dir}/hadoop-${name}-${version}.jar"
      manifest="${root}/META-INF/MANIFEST.MF">
      <manifest>
     <attribute name="Bundle-ClassPath"
        value="classes/, 
 lib/hadoop-mapreduce-client-core-${hadoop.version}.jar,
 lib/hadoop-mapreduce-client-common-${hadoop.version}.jar,
 lib/hadoop-mapreduce-client-jobclient-${hadoop.version}.jar,
 lib/hadoop-auth-${hadoop.version}.jar,
 lib/hadoop-common-${hadoop.version}.jar,
 lib/hadoop-hdfs-${hadoop.version}.jar,
 lib/protobuf-java-${protobuf.version}.jar,
 lib/log4j-${log4j.version}.jar,
 + lib/commons-collections-${commons-collections.version}.jar,
 =lib/commons-cli-1.2.jar,
 =lib/commons-configuration-1.6.jar,
 =lib/commons-httpclient-3.1.jar,
 =lib/commons-lang-2.5.jar,
 =lib/jackson-core-asl-1.8.8.jar,
 =lib/jackson-mapper-asl-1.8.8.jar,
 =lib/slf4j-log4j12-1.7.5.jar,
 =lib/slf4j-api-1.7.5.jar,
 lib/guava-${guava.version}.jar,
 lib/netty-${netty.version}.jar"/>
     </manifest>
      <fileset dir="${build.dir}" includes="classes/ lib/"/>
      <!--fileset dir="${build.dir}" includes="*.xml"/-->
      <fileset dir="${root}" includes="resources/ plugin.xml"/>
    </jar>
  </target>
 
</project>

以上以标有“=”的部分我修改为

lib/commons-cli-${commons-cli.version}.jar,
lib/commons-configuration-${commons-configuration.version}.jar,
lib/commons-httpclient-${commons-httpclient.version}.jar,
lib/commons-lang-${commons-lang.version}.jar,
lib/jackson-core-asl-${jackson.version},
lib/jackson-mapper-asl-${jackson.version}.jar,
lib/slf4j-log4j12-${slf4j-log4j12.version}.jar,
lib/slf4j-api-${slf4j-api.version}.jar,
标有“+”的部分是我加上的。

makePlus.sh

我的路径为/usr/local/hadoop/hadoop2x-eclipse-plugin-master/src/contrib/eclipse-plugin/makePlus.sh

原内容:

ant jar -Dversion=2.0.4 -Declipse.home=/opt/eclipse -Dhadoop.home=/usr/share/hadoop
修改后
ant jar -Dversion=2.5.2 -Declipse.home=/home/hadoop/eclipse -Dhadoop.home=/home/hadoop/hadoop-2.5.2
使之可以执行
chmod +X makePlus.sh

3.编译

进入makePlus.sh所在目录,执行我们修改好的makePlus.sh文件 ./makePlus.sh
在这过程中有一步ivy-resolve-common很长时间没反应,我不清楚是死掉了还是在运行中所以就一直等啊等,上网查说这一步是很花时间的我也就没有在意,
可是都一下午了还是没有任何提示成功或者失败。我这一想有可能是网络不通,可是我的虚拟机是能联着网的,只有一种可能了,就是编译过程中需求去外国的网站下载所依赖的包,需要翻墙。想到这里我立马开个翻墙软件,再次执行 ./makePlus.sh,奇迹出现了!哈哈 编译成功了
Buildfile: /home/hadoop/hadoop2x-eclipse-plugin/src/contrib/eclipse-plugin/build.xml
 
check-contrib:
 
init:
     [echo] contrib: eclipse-plugin
 
init-contrib:
 
ivy-download:
      [get] Getting: http://repo2.maven.org/maven2/org/apache/ivy/ivy/2.1.0/ivy-2.1.0.jar
      [get] To: /home/hadoop/hadoop2x-eclipse-plugin/ivy/ivy-2.1.0.jar
      [get] Not modified - so not downloaded
 
ivy-probe-antlib:
 
ivy-init-antlib:
 
ivy-init:
[ivy:configure] :: Ivy 2.1.0 - 20090925235825 :: http://ant.apache.org/ivy/ ::
[ivy:configure] :: loading settings :: file = /home/hadoop/hadoop2x-eclipse-plugin/ivy/ivysettings.xml
 
ivy-resolve-common:
     [echo] loglevel:quiet;ant.project.name:eclipse-plugin
 
ivy-retrieve-common:
[ivy:cachepath] DEPRECATED: 'ivy.conf.file' is deprecated, use 'ivy.settings.file' instead
[ivy:cachepath] :: loading settings :: file = /home/hadoop/hadoop2x-eclipse-plugin/ivy/ivysettings.xml
 
compile:
     [echo] contrib: eclipse-plugin
    [javac] /home/hadoop/hadoop2x-eclipse-plugin/src/contrib/eclipse-plugin/build.xml:76: warning: 'includeantruntime' was not set, defaulting to build.sysclasspath=last; set to false for repeatable builds
 
jar:
     [copy] Copying 1 file to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/jackson-core-asl-1.9.13.jar to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/jackson-core-asl-1.9.13.jar
     [copy] Copying 1 file to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/jackson-mapper-asl-1.9.13.jar to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/jackson-mapper-asl-1.9.13.jar
     [copy] Copying 1 file to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/slf4j-log4j12-1.7.5.jar to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/slf4j-log4j12-1.7.5.jar
     [copy] Copying 1 file to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/slf4j-api-1.7.5.jar to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/slf4j-api-1.7.5.jar
     [copy] Copying 1 file to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/guava-11.0.2.jar to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/guava-11.0.2.jar
     [copy] Copying 1 file to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/hadoop-auth-2.5.2.jar to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/hadoop-auth-2.5.2.jar
     [copy] Copying 1 file to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib
     [copy] Copying /home/hadoop/hadoop-2.5.2/share/hadoop/common/lib/netty-3.6.2.Final.jar to /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/lib/netty-3.6.2.Final.jar
   <span style="font-size:18px;"></span>  [jar] Building jar: /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/hadoop-eclipse-plugin-2.5.2.jar
 
BUILD SUCCESSFUL
Total time: 16 seconds
 
log4j:WARN No appenders could be found for logger (org.apache.hadoop.util.Shell).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
[jar] Building jar: /home/hadoop/hadoop2x-eclipse-plugin/build/contrib/eclipse-plugin/hadoop-eclipse-plugin-2.5.2.jar就是编译好的jar包所在路径了!

4.测试

把编译好的hadoop-eclipse-plugin-2.5.2.jar放在eclipse的plugins目录下,重启eclipse,配置好后发现不能 new hadoop location,现象是点击后没有任何反应。只好退出eclipse 在终端里进入eclipse 目录执行

./eclipse -clean -consolelog -debug
重新进入eclipse 点击 new hadoop location,查看终端里报出的异常信息java.lang.NoClassDefFoundError

build.xml添加绿色部分重新编译如下所示:

放在eclipse的plugins目录下重启eclipse,便可以使用了


1
0

查看评论
* 以上用户言论只代表其个人观点,不代表CSDN网站的观点或立场
    个人资料
    • 访问:1505796次
    • 积分:21473
    • 等级:
    • 排名:第368名
    • 原创:716篇
    • 转载:4篇
    • 译文:0篇
    • 评论:258条
    博客专栏
    Dubbo

    文章:20篇

    阅读:24150
    JMS

    文章:13篇

    阅读:7154
    Netty

    文章:1篇

    阅读:1021
    Redis

    文章:18篇

    阅读:68404
    Memcached

    文章:9篇

    阅读:33259
    设计模式

    文章:22篇

    阅读:39372
    Mycat

    文章:18篇

    阅读:15172
    Web前端

    文章:22篇

    阅读:65624
    Linux

    文章:76篇

    阅读:165012
    JVM

    文章:15篇

    阅读:41178
    J2EE

    文章:72篇

    阅读:213775
    Java

    文章:151篇

    阅读:369149
    Hadoop

    文章:88篇

    阅读:143486
    MySQL

    文章:128篇

    阅读:188107
    最新评论