倒排索引的简化版

简单介绍下倒排索引：它广泛的应用于全文搜索引擎，提供了一种根据内容查找文档的方式，一般情况下，我们是知道文档才会知道文档的内容，而搜索是依靠关键字进行查找的，这与一般的情况相反，所以称为倒排索引。

现在我们的任务是：

已知三个文档：

file1: mapreduce is simple
file2: mapreduce is powerful is simple
file3: hello mapreduce bye mapreduce

对其进行倒排索引，得到结果如下：

bye    file3.txt:1
hello    file3.txt:1
is    file1.txt:1
is    file2.txt:2
mapreduce    file1.txt:1
mapreduce    file2.txt:1
mapreduce    file3.txt:2
powerful    file2.txt:1
simple    file1.txt:1
simple    file2.txt:1

代码如下：

import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;


public class InvertedIndex extends Configured implements Tool{

	enum Counter{
		LINERSKIP;
	}
	
	public static class Map extends Mapper<LongWritable,Text,Text,Text>  //static is necessary
	{	
		public void map(LongWritable key,Text value,Context context)throws IOException,InterruptedException
		{	
			try{
				String[] linesplit = value.toString().split(" ");
				int n = linesplit.length;
				
				for(int i = 1; i < n; i++)
				{
					String keyinfo = linesplit[i];
					keyinfo = keyinfo + ":" + linesplit[0].substring(0, linesplit[0].length()-1) + ".txt";
					context.write(new Text(keyinfo),new Text("1"));
				}
			}
			catch(java.lang.ArrayIndexOutOfBoundsException e){
				context.getCounter(Counter.LINERSKIP).increment(1);
				return;
			}
			
		
		}
	}
	
	public static class Combine extends Reducer<Text,Text,Text,Text>
	{	
		public void reduce(Text key,Iterable<Text> values,Context context) throws IOException,InterruptedException
		{
			int temp = 0;
			for(Text value : values)
				temp += Integer.parseInt(value.toString());
			int x = key.toString().indexOf(":");
			String keyinfo = key.toString().substring(0, x);
			String valueinfo = key.toString().substring(x+1) + ":" + Integer.valueOf(temp).toString();
			context.write(new Text(keyinfo),new Text(valueinfo));
		}
	}
	
	public static class Reduce extends Reducer<Text,Text,Text,Text>
	{
		
		public void reduce(Text key,Iterable<Text> values,Context context) throws IOException,InterruptedException
		{
			String str = "";
			for(Text value : values)
			{
				str  = str + value.toString() + ";";
			}
			context.write(key, new Text(str));
		}
	}
	

	public int run(String[] args) throws Exception 
	{
		Configuration conf = getConf();
		Job job = new Job(conf,"InvertedIndex");
		job.setJarByClass(InvertedIndex.class);
		
		FileInputFormat.addInputPath(job, new Path(args[0]));
		FileOutputFormat.setOutputPath(job, new Path(args[1]));
		
		job.setMapperClass(Map.class);
		job.setCombinerClass(Combine.class);
		job.setReducerClass(Reduce.class);
		job.setOutputKeyClass(Text.class);
		job.setOutputValueClass(Text.class);
	//	job.setOutputValueClass(IntWritable.class);
		job.waitForCompletion(true);
		return job.isSuccessful() ? 0 : 1;
	}
	

	
	public static void main(String[] args) throws Exception
	{
		
		int res = ToolRunner.run(new Configuration(),new InvertedIndex() ,args);
		System.exit(res);
	}
	

}