基于Map-Reduce的选择运算

Selections really do not need the full power of MapReduce. They can be donemost conveniently in the map portion alone, although they could also be donein the reduce portion alone. Here is a MapReduce implementation of selectionσC (R).

The Map Function: For each tuple t in R, test if it satisfies C. If so, producethe key-value pair (t, t). That is, both the key and value are t.

The Reduce Function: The Reduce function is the identity. It simply passeseach key-value pair to the output.

Note that the output is not exactly a relation, because it has key-value pairs.However, a relation can be obtained by using only the value components (oronly the key components) of the output.

下边这段代码进行了选择预算，选取性别为女的记录（输入文件中每行为一条记录，均含有性别）

代码如下：

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;

public class Selection {

  public static class TokenizerMapper
       extends Mapper<Object, Text, Text, Text>{

    private Text word = new Text();

    public void map(Object key, Text value, Context context
                    ) throws IOException, InterruptedException {
      StringTokenizer tokenizerArticle = new StringTokenizer(value.toString(),"\n");//将整个文本按行进行切分
      while (tokenizerArticle.hasMoreTokens()) {
                                String line=tokenizerArticle.nextToken();//获取行
                                StringTokenizer tokenizerLine = new StringTokenizer(line);//处理行
                                while (tokenizerLine.hasMoreTokens()) {
                                        if(tokenizerLine.nextToken().equals("女")){
                                                word.set(line);//满足条件时将行赋给word
                                                context.write(word, word);//并输出
                                        }
                                }
                        }
                 }
         }

 public static class SelectionReducer
       extends Reducer<Text,Text,Text,Text> {

    public void reduce(Text key, Iterable<Text> values,
                       Context context
                      ) throws IOException, InterruptedException {
      context.write(key, new Text(""));//将Map传来的值直接输出,vaules被忽略
    }
  }

  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
    if (otherArgs.length < 2) {
      System.err.println("Usage: wordcount <in> [<in>...] <out>");
      System.exit(2);
    }
    Job job = new Job(conf, "Selection");
    job.setJarByClass(Selection.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(SelectionReducer.class);
    job.setReducerClass(SelectionReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    for (int i = 0; i < otherArgs.length - 1; ++i) {
      FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
    }
    FileOutputFormat.setOutputPath(job,
      new Path(otherArgs[otherArgs.length - 1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);
  }
}