编写一个倒排索引程序

使HDFS上test文件夹下的文件执行此程序后结果显示如下：

 

affairs -->file:work.txt,cp:1

all -->file:work.txt,cp:1

and -->file:work.txt,cp:4

are -->file:work.txt,cp:3

areas   -->file:work.txt,cp:1

be  -->file:work.txt,cp:1

been    -->file:work.txt,cp:1

bright  -->file:work.txt,cp:1

but -->file:work.txt,cp:1

careful -->file:work.txt,cp:1

cities  -->file:work.txt,cp:2

city    -->file:work.txt,cp:2

 

package kgc;
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class InvertedIndex {
public static class Map extends Mapper<LongWritable, Text, Text, Text>{
private Text keyInfo=new Text();//用来存放输出的key
private Text valueInfo=new Text();//用来存放输出的value
private FileSplit split;
private String pattern="[^\\w]";//正则表达式

@Override
protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
split=(FileSplit) context.getInputSplit();
String valueStr=value.toString();
valueStr=valueStr.replaceAll(pattern, " ");
StringTokenizer st=new StringTokenizer(valueStr);
while(st.hasMoreTokens()){
keyInfo.set(st.nextToken()+":"+split.getPath().getName().toString());
System.out.println(keyInfo);
valueInfo.set("1");
context.write(keyInfo, valueInfo);
}
}
}

public static class Combiner extends Reducer<Text, Text, Text, Text>{
private Text valueInfo=new Text();
@Override
protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
int sum=0;
for(Text val:values){
sum=sum+Integer.parseInt(val.toString());
}
int keySplitIndex=key.toString().indexOf(":");
String keyWord=key.toString().substring(0, keySplitIndex);
valueInfo.set("-->file:"+key.toString().substring(keySplitIndex+1, key.toString().length())+",cp:"+sum);
key.set(keyWord);
context.write(key, valueInfo);
System.out.println("key:"+key+"value:"+valueInfo);
}
}

public static class Reduce extends Reducer<Text, Text, Text, Text>{
private Text valueInfo=new Text();

@Override
protected void reduce(Text key, Iterable<Text> values, Reducer<Text, Text, Text, Text>.Context context)
throws IOException, InterruptedException {
String textList=new String();
int i=0;
for(Text val:values){
if(textList.equals("")){
textList=textList+val.toString();
}else{
textList=textList+";"+val.toString();
}
++i;
System.out.println("textList:"+i+":"+textList+"key:"+key);
}
valueInfo.set(textList);
context.write(key, valueInfo);
}

}
public static void main(String[] args) throws Exception {
Configuration conf=new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.9.154:8020/");
Job job=new Job(conf);
job.setJarByClass(InvertedIndex.class);
job.setMapperClass(Map.class);
job.setCombinerClass(Combiner.class);
job.setReducerClass(Reduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);

FileInputFormat.addInputPath(job, new Path("hdfs://192.168.9.154:8020/test/work.txt"));
FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.9.154:8020/output/test"));
System.exit(job.waitForCompletion(true)?0:1);
}

}