Hadoop下进行反向索引(Inverted Index)操作

 今天上网无意中看到了这篇文章，里面的效果不错，但是代码自己还没有验证过。想先留下来。这篇为转载，等自己调试代码通过后再发表个原创版的！

 

原版地址：http://blog.csdn.net/xw13106209/article/details/6123407

参考资料：
代码参考1：http://www.pudn.com/downloads212/sourcecode/unix_linux/detail999273.html
理论参考2：http://zhangyu8374.javaeye.com/blog/86307，http://nything.javaeye.com/blog/411787

在eclipse下创建map/reduce项目InvertedIndex，然后将参考1中的src目录拷贝到项目目录下替换原有src目录。

在本地创建文件夹IndexTest并在里面创建3个文件，每个文件中的内容如下。
    * T0 = "it is what it is"
    * T1 = "what is it"
    * T2 = "it is a banana"
其中T0，T1，T2分别是文件名，后面为文件内容。将IndexTest文件夹上传到DFS中。然后运行反向索引程序。

最后输出结果为：
a     (T2, 3)
banana     (T2, 4)
is     (T2, 2) (T0, 2) (T0, 5) (T1, 2)
it     (T1, 3) (T2, 1) (T0, 1) (T0, 4)
what     (T0, 3) (T1, 1)

 

代码清单：
InvertedIndex.java

view plain copy to clipboard print ?

1. /* 
2.  * To change this template, choose Tools | Templates 
3.  * and open the template in the editor. 
4.  */  
5. package pa4;  
6. import java.io.IOException;  
7. import org.apache.hadoop.conf.Configuration;  
8. import org.apache.hadoop.fs.FileSystem;  
9. import org.apache.hadoop.fs.Path;  
10. import org.apache.hadoop.io.Text;  
11. import org.apache.hadoop.mapreduce.Job;  
12. import org.apache.hadoop.mapreduce.Mapper;  
13. import org.apache.hadoop.mapreduce.Reducer;  
14. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
15. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
16. import org.apache.hadoop.util.GenericOptionsParser;  
17. /** 
18.  * 
19.  * @author Ming 
20.  */  
21. public class InvertedIndex {  
22.     public static class TokenizerMapper  
23.         extends Mapper<Text, ValuePair, Text, ValuePair> {  
24.     @Override  
25.     public void map(Text key, ValuePair value, Context context) throws IOException, InterruptedException {  
26.     // TokenInputFormat has generate (word, (fileID, wordPosition))  
27.     // so mapper just spill it to reducer  
28.         key.set(key.toString().toLowerCase());  
29.         context.write(key, value);  
30.     }  
31.     }  
32.     public static class IndexReducer  
33.         extends Reducer<Text, ValuePair, Text, Text> {  
34.     private Text postings = new Text();  
35.     @Override  
36.     public void reduce(Text key, Iterable<ValuePair> values,  
37.         Context context) throws IOException, InterruptedException {  
38.         String list = "";  
39.         for (ValuePair val : values) {  
40.         list += " " + val.toString();  
41.         }  
42.         postings.set(list);  
43.         context.write(key, postings);  
44.     }  
45.     }  
46.     public static void main(String[] args) throws Exception {  
47.     Configuration conf = new Configuration();  
48.     String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();  
49.     if (otherArgs.length != 2) {  
50.         System.err.println("Usage: InvertedIndex <in-dir> <out-dir>");  
51.         System.exit(2);  
52.     }  
53.     // remove the old output dir  
54.     FileSystem.get(conf).delete(new Path(otherArgs[1]), true);  
55.     Job job = new Job(conf, "Inverted Indexer");  
56.     job.setJarByClass(InvertedIndex.class);  
57.     job.setInputFormatClass(TokenInputFormat.class);  
58.     job.setMapperClass(InvertedIndex.TokenizerMapper.class);  
59.     //job.setCombinerClass(InvertedIndex.IndexReducer.class);  
60.     job.setReducerClass(InvertedIndex.IndexReducer.class);  
61.       
62.     job.setMapOutputKeyClass(Text.class);  
63.     job.setMapOutputValueClass(ValuePair.class);  
64.     job.setOutputKeyClass(Text.class);  
65.     job.setOutputValueClass(Text.class);  
66.     FileInputFormat.addInputPath(job, new Path(otherArgs[0]));  
67.     FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));  
68.     System.exit(job.waitForCompletion(true) ? 0 : 1);  
69.     }  
70. }  

/* * To change this template, choose Tools | Templates * and open the template in the editor. */ package pa4; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.util.GenericOptionsParser; /** * * @author Ming */ public class InvertedIndex { public static class TokenizerMapper extends Mapper<Text, ValuePair, Text, ValuePair> { @Override public void map(Text key, ValuePair value, Context context) throws IOException, InterruptedException { // TokenInputFormat has generate (word, (fileID, wordPosition)) // so mapper just spill it to reducer key.set(key.toString().toLowerCase()); context.write(key, value); } } public static class IndexReducer extends Reducer<Text, ValuePair, Text, Text> { private Text postings = new Text(); @Override public void reduce(Text key, Iterable<ValuePair> values, Context context) throws IOException, InterruptedException { String list = ""; for (ValuePair val : values) { list += " " + val.toString(); } postings.set(list); context.write(key, postings); } } public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs(); if (otherArgs.length != 2) { System.err.println("Usage: InvertedIndex <in-dir> <out-dir>"); System.exit(2); } // remove the old output dir FileSystem.get(conf).delete(new Path(otherArgs[1]), true); Job job = new Job(conf, "Inverted Indexer"); job.setJarByClass(InvertedIndex.class); job.setInputFormatClass(TokenInputFormat.class); job.setMapperClass(InvertedIndex.TokenizerMapper.class); //job.setCombinerClass(InvertedIndex.IndexReducer.class); job.setReducerClass(InvertedIndex.IndexReducer.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(ValuePair.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); FileInputFormat.addInputPath(job, new Path(otherArgs[0])); FileOutputFormat.setOutputPath(job, new Path(otherArgs[1])); System.exit(job.waitForCompletion(true) ? 0 : 1); } }

 

TokenInputFormat.java

view plain copy to clipboard print ?

1. package pa4;  
2. import java.io.IOException;  
3. import org.apache.hadoop.fs.FileSystem;  
4. import org.apache.hadoop.fs.Path;  
5. import org.apache.hadoop.io.Text;  
6. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
7. import org.apache.hadoop.mapreduce.lib.input.FileSplit;  
8. import org.apache.hadoop.mapreduce.InputSplit;  
9. import org.apache.hadoop.conf.Configuration;  
10. import org.apache.hadoop.fs.FSDataInputStream;  
11. import org.apache.hadoop.mapreduce.JobContext;  
12. import org.apache.hadoop.mapreduce.RecordReader;  
13. import org.apache.hadoop.mapreduce.TaskAttemptContext;  
14. import org.apache.hadoop.mapreduce.TaskAttemptID;  
15. import org.apache.hadoop.util.LineReader;  
16. import java.util.StringTokenizer;  
17. public class TokenInputFormat extends FileInputFormat<Text, ValuePair> {  
18.     /** 
19.      * Don't allow the files to be split! 
20.      */  
21.     @Override  
22.     protected boolean isSplitable(JobContext ctx, Path filename) {  
23.     // ensure the input files are not splittable!  
24.     return false;  
25.     }  
26.     /** 
27.      * Just return the record reader 
28.      * key is the docno 
29.      */  
30.     public RecordReader<Text, ValuePair> createRecordReader(InputSplit split,  
31.         TaskAttemptContext ctx)  
32.         throws IOException, InterruptedException {  
33.     return new TokenRecordReader();  
34.     }  
35.     public static class TokenRecordReader extends RecordReader<Text, ValuePair> {  
36.     private long start;  
37.     private long pos;  
38.     private long end;  
39.     private LineReader in;  
40.     private int maxLineLength;  
41.     private Text line;  
42.     private Text key = null;  
43.     private ValuePair value = null;  
44.     private StringTokenizer tokens = null;  
45.     private int tokenPos = 0;  
46.     private String fileID = "0";    // input file id that appears in inverted index  
47.     public void initialize(InputSplit genericSplit,  
48.         TaskAttemptContext context) throws IOException {  
49.         FileSplit split = (FileSplit) genericSplit;  
50.         Configuration job = context.getConfiguration();  
51.         this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength",  
52.             Integer.MAX_VALUE);  
53.         start = split.getStart();  
54.         end = start + split.getLength();  
55.         final Path file = split.getPath();  
56.         // Assume file name is an integer of file ID  
57.         fileID = file.getName();  
58.         FileSystem fs = file.getFileSystem(job);  
59.         FSDataInputStream fileIn = fs.open(split.getPath());  
60.         in = new LineReader(fileIn, job);  
61.         this.pos = start;  
62.         line = new Text();  
63.         key = new Text();  
64.         value = new ValuePair();  
65.     }  
66.     public boolean nextKeyValue() throws IOException {  
67.         boolean splitEnds = false;  
68.         while (tokens == null || !tokens.hasMoreTokens()) {  
69.         int lineSize = in.readLine(line, maxLineLength,  
70.             Math.max((int) Math.min(Integer.MAX_VALUE, end - pos),  
71.             maxLineLength));  
72.         if (lineSize == 0) {  
73.             splitEnds = true;  
74.             break;  
75.         }  
76.         pos += lineSize;  
77.         tokens = new StringTokenizer(line.toString(), " /t/n/r/f,.;<>-?///!'/":=*{}()$[]");  
78.         }  
79.         if (splitEnds) {  
80.         key = null;  
81.         value = null;  
82.         line = null;  
83.         tokens = null;  
84.         return false;  
85.         } else  
86.         return true;  
87.     }  
88.     @Override  
89.     public Text getCurrentKey() {  
90.         key.set(tokens.nextToken());  
91.         tokenPos ++;  
92.         return key;  
93.     }  
94.     @Override  
95.     public ValuePair getCurrentValue() {  
96.         value.set(fileID, tokenPos);  
97.         return value;  
98.     }  
99.     /** 
100.      * Get the progress within the split 
101.      */  
102.     public float getProgress() {  
103.         if (start == end) {  
104.         return 0.0f;  
105.         } else {  
106.         return Math.min(1.0f, (pos - start) / (float) (end - start));  
107.         }  
108.     }  
109.     public synchronized void close() throws IOException {  
110.         if (in != null) {  
111.         in.close();  
112.         }  
113.     }  
114.     }  
115.     public static void main(String[] args)  
116.         throws IOException {  
117.     String fn = args[0];  
118.     Configuration conf = new Configuration();  
119.     FileSplit split = new FileSplit(new Path(fn), 0, 10000000, null);  
120.     TokenRecordReader irr = new TokenRecordReader();  
121.     TaskAttemptContext ctx = new TaskAttemptContext(conf,  
122.         new TaskAttemptID("hello", 12, true, 12, 12));  
123.     irr.initialize(split, ctx);  
124.     while (irr.nextKeyValue()) {  
125.         System.out.println(irr.getCurrentKey() + ": " + irr.getCurrentValue());  
126.     }  
127.     }  
128. }  

package pa4; import java.io.IOException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.util.LineReader; import java.util.StringTokenizer; public class TokenInputFormat extends FileInputFormat<Text, ValuePair> { /** * Don't allow the files to be split! */ @Override protected boolean isSplitable(JobContext ctx, Path filename) { // ensure the input files are not splittable! return false; } /** * Just return the record reader * key is the docno */ public RecordReader<Text, ValuePair> createRecordReader(InputSplit split, TaskAttemptContext ctx) throws IOException, InterruptedException { return new TokenRecordReader(); } public static class TokenRecordReader extends RecordReader<Text, ValuePair> { private long start; private long pos; private long end; private LineReader in; private int maxLineLength; private Text line; private Text key = null; private ValuePair value = null; private StringTokenizer tokens = null; private int tokenPos = 0; private String fileID = "0"; // input file id that appears in inverted index public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException { FileSplit split = (FileSplit) genericSplit; Configuration job = context.getConfiguration(); this.maxLineLength = job.getInt("mapred.linerecordreader.maxlength", Integer.MAX_VALUE); start = split.getStart(); end = start + split.getLength(); final Path file = split.getPath(); // Assume file name is an integer of file ID fileID = file.getName(); FileSystem fs = file.getFileSystem(job); FSDataInputStream fileIn = fs.open(split.getPath()); in = new LineReader(fileIn, job); this.pos = start; line = new Text(); key = new Text(); value = new ValuePair(); } public boolean nextKeyValue() throws IOException { boolean splitEnds = false; while (tokens == null || !tokens.hasMoreTokens()) { int lineSize = in.readLine(line, maxLineLength, Math.max((int) Math.min(Integer.MAX_VALUE, end - pos), maxLineLength)); if (lineSize == 0) { splitEnds = true; break; } pos += lineSize; tokens = new StringTokenizer(line.toString(), " /t/n/r/f,.;<>-?///!'/":=*{}()$[]"); } if (splitEnds) { key = null; value = null; line = null; tokens = null; return false; } else return true; } @Override public Text getCurrentKey() { key.set(tokens.nextToken()); tokenPos ++; return key; } @Override public ValuePair getCurrentValue() { value.set(fileID, tokenPos); return value; } /** * Get the progress within the split */ public float getProgress() { if (start == end) { return 0.0f; } else { return Math.min(1.0f, (pos - start) / (float) (end - start)); } } public synchronized void close() throws IOException { if (in != null) { in.close(); } } } public static void main(String[] args) throws IOException { String fn = args[0]; Configuration conf = new Configuration(); FileSplit split = new FileSplit(new Path(fn), 0, 10000000, null); TokenRecordReader irr = new TokenRecordReader(); TaskAttemptContext ctx = new TaskAttemptContext(conf, new TaskAttemptID("hello", 12, true, 12, 12)); irr.initialize(split, ctx); while (irr.nextKeyValue()) { System.out.println(irr.getCurrentKey() + ": " + irr.getCurrentValue()); } } }

 

ValuePair.java

view plain copy to clipboard print ?

1. package pa4;  
2. /* 
3.  * To change this template, choose Tools | Templates 
4.  * and open the template in the editor. 
5.  */  
6. import java.io.*;  
7. import org.apache.hadoop.io.*;  
8. /** 
9.  * 
10.  * @author Ming 
11.  */  
12. public class ValuePair implements WritableComparable<ValuePair> {  
13.     private Text one;  
14.     private IntWritable two;  
15.     public void set(Text first, IntWritable second) {  
16.     one = first;  
17.     two = second;  
18.     }  
19.     public void set(String first, int second) {  
20.     one.set(first);  
21.     two.set(second);  
22.     }  
23.     public ValuePair() {  
24.     set(new Text(), new IntWritable());  
25.     }  
26.     public ValuePair(Text first, IntWritable second) {  
27.     set(first, second);  
28.     }  
29.     public ValuePair(String first, int second) {  
30.     set(first, second);  
31.     }  
32.     public Text getFirst() {  
33.     return one;  
34.     }  
35.     public IntWritable getSecond() {  
36.     return two;  
37.     }  
38.     @Override  
39.     public void write(DataOutput out) throws IOException {  
40.     one.write(out);  
41.     two.write(out);  
42.     }  
43.     @Override  
44.     public void readFields(DataInput in) throws IOException {  
45.     one.readFields(in);  
46.     two.readFields(in);  
47.     }  
48.     @Override  
49.     public int hashCode() {  
50.     return one.hashCode();  
51.     }  
52.     @Override  
53.     public boolean equals(Object o) {  
54.     if (o instanceof ValuePair) {  
55.         ValuePair tp = (ValuePair)o;  
56.         return one.equals(tp.one);  
57.     }  
58.     return false;  
59.     }  
60.     @Override  
61.     public String toString() {  
62.     return "(" + one + ", " + two + ")";  
63.     }  
64.     @Override  
65.     public int compareTo(ValuePair tp) {  
66.     int cmp = one.compareTo(tp.one);  
67.     if (cmp != 0) {  
68.         return cmp;  
69.     }  
70.     return two.compareTo(tp.two);  
71.     }  
72.     public static class Comparator extends WritableComparator {  
73.     private static final Text.Comparator TEXT_COMPARATOR = new Text.Comparator();  
74.     private static final IntWritable.Comparator INT_COMPARATOR = new IntWritable.Comparator();  
75.     public Comparator() {  
76.         super(ValuePair.class);  
77.     }  
78.     @Override  
79.     public int compare(byte[] b1, int s1, int l1,  
80.                 byte[] b2, int s2, int l2) {  
81.         try {  
82.         int oneL1 = WritableUtils.decodeVIntSize(b1[s1]) + readVInt(b1, s1);  
83.         int oneL2 = WritableUtils.decodeVIntSize(b2[s2]) + readVInt(b2, s2);  
84.         int cmp = TEXT_COMPARATOR.compare(b1, s1, oneL1, b2, s2, oneL2);  
85.         if (cmp != 0) {  
86.             return cmp;  
87.         }  
88.         return INT_COMPARATOR.compare(b1, s1+oneL1, l1-oneL1,  
89.                         b2, s2+oneL2, l2-oneL2);  
90.         } catch (IOException e) {  
91.         throw new IllegalArgumentException(e);  
92.         }  
93.     }  
94.     @Override  
95.     public int compare(WritableComparable a, WritableComparable b) {  
96.         if (a instanceof ValuePair && b instanceof ValuePair) {  
97.         return ((ValuePair) a).compareTo((ValuePair) b);  
98.         }  
99.         return super.compare(a, b);  
100.     }  
101.     }  
102.     static {  
103.     WritableComparator.define(ValuePair.class, new Comparator());  
104.     }  
105. }  

package pa4; /* * To change this template, choose Tools | Templates * and open the template in the editor. */ import java.io.*; import org.apache.hadoop.io.*; /** * * @author Ming */ public class ValuePair implements WritableComparable<ValuePair> { private Text one; private IntWritable two; public void set(Text first, IntWritable second) { one = first; two = second; } public void set(String first, int second) { one.set(first); two.set(second); } public ValuePair() { set(new Text(), new IntWritable()); } public ValuePair(Text first, IntWritable second) { set(first, second); } public ValuePair(String first, int second) { set(first, second); } public Text getFirst() { return one; } public IntWritable getSecond() { return two; } @Override public void write(DataOutput out) throws IOException { one.write(out); two.write(out); } @Override public void readFields(DataInput in) throws IOException { one.readFields(in); two.readFields(in); } @Override public int hashCode() { return one.hashCode(); } @Override public boolean equals(Object o) { if (o instanceof ValuePair) { ValuePair tp = (ValuePair)o; return one.equals(tp.one); } return false; } @Override public String toString() { return "(" + one + ", " + two + ")"; } @Override public int compareTo(ValuePair tp) { int cmp = one.compareTo(tp.one); if (cmp != 0) { return cmp; } return two.compareTo(tp.two); } public static class Comparator extends WritableComparator { private static final Text.Comparator TEXT_COMPARATOR = new Text.Comparator(); private static final IntWritable.Comparator INT_COMPARATOR = new IntWritable.Comparator(); public Comparator() { super(ValuePair.class); } @Override public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) { try { int oneL1 = WritableUtils.decodeVIntSize(b1[s1]) + readVInt(b1, s1); int oneL2 = WritableUtils.decodeVIntSize(b2[s2]) + readVInt(b2, s2); int cmp = TEXT_COMPARATOR.compare(b1, s1, oneL1, b2, s2, oneL2); if (cmp != 0) { return cmp; } return INT_COMPARATOR.compare(b1, s1+oneL1, l1-oneL1, b2, s2+oneL2, l2-oneL2); } catch (IOException e) { throw new IllegalArgumentException(e); } } @Override public int compare(WritableComparable a, WritableComparable b) { if (a instanceof ValuePair && b instanceof ValuePair) { return ((ValuePair) a).compareTo((ValuePair) b); } return super.compare(a, b); } } static { WritableComparator.define(ValuePair.class, new Comparator()); } }