1. 首先前往 官网(Hadoop 2.6 下载地址)上下载Hadoop的源码文件,并解压
2. 事先请确定已经安装好jdk以及maven(Maven安装教程 这是其他人写的一篇博文,保存profile后,记得source下)
3. 进入到hadoop-2.6.0-src/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop 目录
可以看到有很多文件夹,找到ipc目录,在里面mkdir protobuf文件夹(还有一丢丢问题,这个后面解决)
4. 安装protobuf
下载地址 : http://pan.baidu.com/s/1pJlZubT 并解压
切换到protobuf文件夹,依次在终端下输入:
. / configure
make
make check
make install
全部执行完后再输入protoc - - version检查是否安装成功。
5. hadoop-2.6.0-src/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/avro/TestAvroSerialization.java显示找不到AvroRecord类
去grepcode下载2.5.0或者更高版本的。放到相应位置
6. cd到 hadoop-2.6.0-src/hadoop-maven-plugins 目录下 使用:mvn intall
7. cd到hadoop-2.6.0-src 目录下 使用mvn eclipse:eclipse -DskipTests
8. 在Eclipse中:File-Import-Existing Projects into Workspace 选择hadoop-2.6.0-src目录导入(推荐换一个workspace再导入,因为project的数目挺多的)
到了这一步,基本上已经没什么问题了,但是还有一些小问题,需要解决
可能遇到的问题:
1.org.apache.hadoop.ipc.protobuf 包下,可能是空包。
去grepcode上搜索TestProtos和TestRpcServiceProtos两个类,我遇到的情况是本地目录已经存在那两个文件,却没有导入。
没有上述两个类,会提示一大片的EmptyRequestProto之类的无法解析
2.会提示com.sum.tool或者javadoc包无法解析
右键选择buildpath -> configure build path -> library -> add library -> ire system 即可
3. hadoop-streaming 工程中的build path 有问题
把source中的hadoop-yarn-server-resourcemanager/conf删除掉,然后Link Source:源码根目录/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf
4. mac下发现classes.jar找不到的问题:
Exception in thread "main" java.lang.AssertionError: Missing tools.jar at: /Library/Java/JavaVirtualMachines/jdk1.7.0_45.jdk/Contents/Home/Classes/classes.jar. Expression: file.exists()
this is as the compiler needs a file named tools.jar but localted at classes.jar under dir Classes but found nothing.so u can do workaround:
解决办法:
cd $JAVA_HOME mkdir Classes cd Classes ln -s ../lib/tools.jar classes.jar
附件:AvroRecord.java hadoop2.6版本
/**
* Autogenerated by Avro
*
* DO NOT EDIT DIRECTLY
*/
package org.apache.hadoop.io.serializer.avro;
@SuppressWarnings("all")
@org.apache.avro.specific.AvroGenerated
public class AvroRecord extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"AvroRecord\",\"namespace\":\"org.apache.hadoop.io.serializer.avro\",\"fields\":[{\"name\":\"intField\",\"type\":\"int\"}]}");
public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
@Deprecated public int intField;
/**
* Default constructor.
*/
public AvroRecord() {}
/**
* All-args constructor.
*/
public AvroRecord(java.lang.Integer intField) {
this.intField = intField;
}
public org.apache.avro.Schema getSchema() { return SCHEMA$; }
// Used by DatumWriter. Applications should not call.
public java.lang.Object get(int field$) {
switch (field$) {
case 0: return intField;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
// Used by DatumReader. Applications should not call.
@SuppressWarnings(value="unchecked")
public void put(int field$, java.lang.Object value$) {
switch (field$) {
case 0: intField = (java.lang.Integer)value$; break;
default: throw new org.apache.avro.AvroRuntimeException("Bad index");
}
}
/**
* Gets the value of the 'intField' field.
*/
public java.lang.Integer getIntField() {
return intField;
}
/**
* Sets the value of the 'intField' field.
* @param value the value to set.
*/
public void setIntField(java.lang.Integer value) {
this.intField = value;
}
/** Creates a new AvroRecord RecordBuilder */
public static org.apache.hadoop.io.serializer.avro.AvroRecord.Builder newBuilder() {
return new org.apache.hadoop.io.serializer.avro.AvroRecord.Builder();
}
/** Creates a new AvroRecord RecordBuilder by copying an existing Builder */
public static org.apache.hadoop.io.serializer.avro.AvroRecord.Builder newBuilder(org.apache.hadoop.io.serializer.avro.AvroRecord.Builder other) {
return new org.apache.hadoop.io.serializer.avro.AvroRecord.Builder(other);
}
/** Creates a new AvroRecord RecordBuilder by copying an existing AvroRecord instance */
public static org.apache.hadoop.io.serializer.avro.AvroRecord.Builder newBuilder(org.apache.hadoop.io.serializer.avro.AvroRecord other) {
return new org.apache.hadoop.io.serializer.avro.AvroRecord.Builder(other);
}
/**
* RecordBuilder for AvroRecord instances.
*/
public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<AvroRecord>
implements org.apache.avro.data.RecordBuilder<AvroRecord> {
private int intField;
/** Creates a new Builder */
private Builder() {
super(org.apache.hadoop.io.serializer.avro.AvroRecord.SCHEMA$);
}
/** Creates a Builder by copying an existing Builder */
private Builder(org.apache.hadoop.io.serializer.avro.AvroRecord.Builder other) {
super(other);
}
/** Creates a Builder by copying an existing AvroRecord instance */
private Builder(org.apache.hadoop.io.serializer.avro.AvroRecord other) {
super(org.apache.hadoop.io.serializer.avro.AvroRecord.SCHEMA$);
if (isValidValue(fields()[0], other.intField)) {
this.intField = data().deepCopy(fields()[0].schema(), other.intField);
fieldSetFlags()[0] = true;
}
}
/** Gets the value of the 'intField' field */
public java.lang.Integer getIntField() {
return intField;
}
/** Sets the value of the 'intField' field */
public org.apache.hadoop.io.serializer.avro.AvroRecord.Builder setIntField(int value) {
validate(fields()[0], value);
this.intField = value;
fieldSetFlags()[0] = true;
return this;
}
/** Checks whether the 'intField' field has been set */
public boolean hasIntField() {
return fieldSetFlags()[0];
}
/** Clears the value of the 'intField' field */
public org.apache.hadoop.io.serializer.avro.AvroRecord.Builder clearIntField() {
fieldSetFlags()[0] = false;
return this;
}
@Override
public AvroRecord build() {
try {
AvroRecord record = new AvroRecord();
record.intField = fieldSetFlags()[0] ? this.intField : (java.lang.Integer) defaultValue(fields()[0]);
return record;
} catch (Exception e) {
throw new org.apache.avro.AvroRuntimeException(e);
}
}
}
}
参考资料:
1. 在mac中导入hadoop2.6.0源代码至eclipse教程
3. hadoop-2.6.0-src源码导入Eclipse
5. 百度知道:导入时可能遇到的错误以及解决办法 (请参考其他1条回答)