关于mapreduce程序之Mapper、Reducer以及Driver驱动的若干框架总结

  第一套：

为旧的API，此时Maper和Reducer为接口，且该接口均在org.apache.hadoop.mapred.*;中

 Mapper：

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.Mapper;

public class ExampleMapper extends MapRecuceBase implements    Mapper<LongWritable,Text,Text,IntWritable>

  {

      public void map(LongWritable key,Text value,OutputCollector<Text,IntWritable> output,Reporterreporter)   throwsIOException

{

.......

     }

}

    Reducer:

import org.apache.hadoop.mapred.MapReduceBase;

import org.apache.hadoop.mapred.Reducer;

  public class ExampleReducer extends MapReduceBase implements 

Reducer<Text,IntWritable,Text,IntWritable>

{

public void reduce(Text key,Iterator<IntWritable>  values,OutputCollector<Text,IntWritable>,Reporter reporter)

{

........

}

}

Driver:

public class ExampleDrvier{

public static void main(String args[]) throws IOException{

JobConf=new JobConf(ExampleDrvier.class);

conf.setJobName(“Example”);

........

JobClient.runJob(conf);

}

}

或者：

public class ExampleDriver{

public int  run (String args[]) throws Exception{

JobConf conf=new JobConf( getConf(), getClass());

conf.setJobName("example");

..........

JobClient.runJob(conf);

return 0;

}

public static void main(String args[]) throws Exception{

int exitCode=ToolRunner.run(new ExampleDriver(),args);

System.exit(exitCode);

}

}

  第二套：

为新的API，此时Maper和Reducer为虚类，且该接口均在org.apache.hadoop.mapreduce.*;中

Mapper:

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Mapper.Context;

public class ExampleMapper extends Mapper<LongWritable,Text,Text,IntWritable>

{

public void map(LongWritable key,Text value,Context context) throwsIOException,InterruptedException

{

..........

}

}

Reducer:

import org.apache.hadoop.mapreduce.Reducer;

  import org.apache.hadoop.mapreduce.Reducer.Context;

class ExampleReducer extends Reducer<Text ,Text,Text,Text>

{

public void reduce(Text key,Iterable<Text> values,Context context) throwsIOException,InterruptedException

{

..........

}

}

Driver:

public class ExampleDriver{

public static void main(String args[]) throws Exception{

Job job=new Job();

job.setJarByClass(ExampleDriver.class);

............

job.setMapperClass();

.............

System.exit(job.waitForCompletion(true)?0:1);

}

}

或者

public class ExampleDriver extends Configured implements Tool

{

public int run(String args[])throws Exception

{

Configuration conf=getConf();

Job job=new Job(conf,"Example"); //任务名

job.setJarByClass(ExampleDriver.class); //指定class

............

job.waitForCompletion(true);

return job.isSuccessful()?0:1;

}

public static void main(String args[]) throws Exception

{

int res=ToolRunner.run(new Configuration(),new ExampleDriver(),args);

System.exit(res);

}

}