Hadoop 自定义数据类型及输入格式

　　Hadoop提供了大量的数据输入类型，如Text和IntWritable．

　　假如我们需要自定义一个数据类型，首先要实现Writable接口，主要包含两个函数readFields和write．如果需要把新的数据类型作为Map函数的key输出的话，在shuffle阶段会有一个排序的过程，需要对key进行比较，那么这里就推荐实现WritableComparable接口，它比Writable接口多一个compareTo函数．

　　自定义数据类型：

　　注意：自定义的数据类型必须要有空的构造函数，且需要调用父类的构造方法．同时建议在空的构造函数里面对属性进行初始化，因为write函数对实例进行序列化，readFields函数进行反序列化，在进行反序列化时，若属性未进行初始化，可能会出现空指针异常（普通数据类型不会出现该异常，当属性为符合数据类型时才会出现异常），因此建议大家在空的构造函数中将所有的属性都进行初始化．toString函数是作为reduce函数输出时用到的．

package pers.kefault.entity;

import java.io.BufferedReader;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;


public class IndexDoc implements  WritableComparable<IndexDoc> {
	
	private int docId;
	private String urlString;
	private String siteName;
	
	public IndexDoc() {
		super();
		// TODO Auto-generated constructor stub
	}

	public IndexDoc(int docId, String urlString, String siteName) {
		super();
		this.docId = docId;
		this.urlString = urlString;
		this.siteName = siteName;
	}
	//复制对象，好烦～～
	public IndexDoc clone()
	{
		return new IndexDoc(getDocId(),getUrlString(),getSiteName());
	}
	
	public void setDocId(int docId) {
		this.docId = docId;
	}

	public void setUrlString(String urlString) {
		this.urlString = urlString;
	}

	public void setSiteName(String siteName) {
		this.siteName = siteName;
	}

	public int getDocId() {
		return docId;
	}

	public String getSiteName() {
		if(siteName == null || siteName == "")
		{
			siteName = urlString.split("/")[0];
		}
		return siteName;
	}

	public String getUrlString() {
		return urlString;
	}
	
	//读取文件，以Text形式返回
	public Text text()
	{
		//设置读取的文件的路径
		String fileName = "/home/monster/spider/" + urlString;
		File file = new File(fileName);
		BufferedReader reader = null;
		String content = "";
		try
		{
			reader = new BufferedReader(new InputStreamReader(new FileInputStream(file),"GB2312"));
			String tempString = null;
			while((tempString = reader.readLine())!=null)
			{
				content += tempString;
			}
			reader.close();
		}catch (Exception e) {
			// TODO: handle exception
			e.printStackTrace();
		}
		return new Text(content); 
	}

	@Override
	public void readFields(DataInput in) throws IOException {
		// TODO Auto-generated method stub
		docId = in.readInt();
		urlString = in.readUTF();
		siteName = urlString.split("/")[0];
	}

	@Override
	public void write(DataOutput out) throws IOException {
		// TODO Auto-generated method stub
		out.writeInt(docId);
		out.writeUTF(urlString);
	}

	@Override
	public int compareTo(IndexDoc o) {
		// TODO Auto-generated method stub
		if(o.docId < docId)
		{
			return 1;
		}else if(o.docId > docId)
		{
			return -1;
		}
		return 0;
	}
	
	@Override
	public String toString() {
		return docId + "," + urlString + "," + siteName;
	}
}

　　

　　自定义数据输入格式：

package pers.kefault.format;

import java.io.IOException;

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import pers.kefault.entity.IndexDoc;

public class IndexDocInputFormat extends FileInputFormat<Text, IndexDoc> {
	@Override
	public RecordReader<Text, IndexDoc> createRecordReader(InputSplit split, TaskAttemptContext context)
			throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		context.setStatus(split.toString());//不知道干嘛的
		IndexDocRecordReader reader = new IndexDocRecordReader(context.getConfiguration());
		return reader;
	}
}

　　自定义数据输入的RecordReader:

package pers.kefault.format;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.LineRecordReader;

import pers.kefault.entity.IndexDoc;

public class IndexDocRecordReader extends RecordReader<Text, IndexDoc> {

	public static final String KEY_VALUE_SEPERATOR = "mapreduce.input.keyvaluelinerecordreader.key.value.separator";
	private final LineRecordReader lineRecordReader;
	private byte separator = (byte) '\t';
	private Text innerValue;
	private Text key;
	private IndexDoc value;
	
	public Class<Text> getKeyClass() { 
		return Text.class;
	}
	
	@Override
	public  void close() throws IOException {
		// TODO Auto-generated method stub
		System.out.println("close()");
		lineRecordReader.close();
	}

	public IndexDocRecordReader(Configuration conf) {
		// TODO Auto-generated constructor stub
		lineRecordReader = new LineRecordReader();
		String sepStr = conf.get(KEY_VALUE_SEPERATOR,"\t");
		this.separator = (byte) sepStr.charAt(0);
	}

	public static int findSeparator(byte[] utf, int start, int length, byte sep) {
		for (int i = start; i < (start + length); i++) {
			if (utf[i] == sep) {
				return i;
			}
		}
		return -1;
	}
	
	public static void setKeyValue(Text key, IndexDoc value, byte[] line,int lineLen, int pos) {
		if (pos == -1) {
			key.set(line, 0, lineLen);
			value.setDocId(-1);
			value.setSiteName("www.kefault.com");
			value.setUrlString("www.kefault.com/null");
		} else {
			key.set(line, 0, pos); //设置键  从 第 0位置 到 截取标识符的位置
			Text text = new Text();
			text.set(line, pos + 1, lineLen - pos - 1);
			String[] str = text.toString().split(",");
			value.setDocId(Integer.parseInt(str[0]));
			value.setUrlString(str[1]);
			value.setSiteName(str[2]);
		}
	}
	
	@Override
	public Text getCurrentKey() throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		return key;
	}

	@Override
	public IndexDoc getCurrentValue() throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		return value;
	}

	@Override
	public float getProgress() throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		return lineRecordReader.getProgress();
	}

	@Override
	public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		lineRecordReader.initialize(genericSplit, context);
	}

	@Override
	public  boolean nextKeyValue() throws IOException, InterruptedException {
		// TODO Auto-generated method stub
		byte[] line = null;
		int lineLen = -1;
		
		if (lineRecordReader.nextKeyValue()) {
			innerValue = lineRecordReader.getCurrentValue();
			line = innerValue.getBytes();
			lineLen = innerValue.getLength();
		} else {
			System.out.println("return false;");
			return false;
		}
		if (line == null){
			System.out.println("return false;");
			return false;
		}
		
		if (key == null) {
			key = new Text();
		}
		if (value == null) {
			value = new IndexDoc(); 
		}
		
		int pos = findSeparator(line, 0, lineLen, this.separator);
		setKeyValue(key, value, line, lineLen, pos);
		return true;
	}
}

　　整个程序的重点放在RecordReader上．简单来说，读取数据按行读入，返回一个key-value对．而开始读入的信息保存成Text类型，然后对Text解析，变成key-value对的形式．别看代码这么多，其实拿过来要修改的地方也不是很多．对Text分解时，依据的是一个分割符号，这里用的是'\t'，大家也可以自定．然后依据分割成的字符串创建key的对象和value的对象就行了．

　　如果大家使用Eclipse的话，在控制台是不显示MapReduce执行过程中的错误的，所以大家可以看一下点击打开链接．希望可以让你调试代码更轻松一点．