mapreduce作业单元测试

用官方提供的MRunit包进行对MapReducer作业进行测试

http://mrunit.apache.org/

先要下载对应的测试包，分为1.0和2.0的hadoop版本

 

来自：

https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial

 

Mapper

 

Java代码  

1. import org.apache.hadoop.io.IntWritable;   
2. import org.apache.hadoop.io.LongWritable;   
3. import org.apache.hadoop.io.Text;   
4. import org.apache.hadoop.mapreduce.Mapper;   
5.   
6. public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>   
7. {   
8.   
9.     private Text status = new Text();   
10.   
11.     private final static IntWritable addOne = new IntWritable(1);   
12.   
13.     @Override  
14.     protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException   
15.     {   
16.   
17.         //655209;1;796764372490213;804422938115889;6 is the Sample record format   
18.         String[] line = value.toString().split(";");   
19.         // If record is of SMS CDR   
20.         if (Integer.parseInt(line[1]) == 1)   
21.         {   
22.             status.set(line[4]);   
23.             context.write(status, addOne);   
24.         }   
25.     }   
26.   
27. }  

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class SMSCDRMapper extends Mapper<LongWritable, Text, Text, IntWritable>
{

    private Text status = new Text();

    private final static IntWritable addOne = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Context context) throws java.io.IOException, InterruptedException
    {

        //655209;1;796764372490213;804422938115889;6 is the Sample record format
        String[] line = value.toString().split(";");
        // If record is of SMS CDR
        if (Integer.parseInt(line[1]) == 1)
        {
            status.set(line[4]);
            context.write(status, addOne);
        }
    }

}

 

 

Reducer:

 

 

Java代码  

1. import java.util.List;   
2.   
3. import org.apache.hadoop.io.IntWritable;   
4. import org.apache.hadoop.io.LongWritable;   
5. import org.apache.hadoop.io.Text;   
6. import org.apache.hadoop.mrunit.mapreduce.MapDriver;   
7. import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;   
8. import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;   
9. import org.junit.Before;   
10. import org.junit.Test;   
11.   
12. import flex.messaging.io.ArrayList;   
13.   
14. public class SMSCDRMapperReducerTest   
15. {   
16.     MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;   
17.     ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;   
18.     MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;   
19.     @Before  
20.     public void setup()   
21.     {   
22.         SMSCDRMapper mapper = new SMSCDRMapper();   
23.         SMSCDRReducer reducer = new SMSCDRReducer();   
24.         mapDriver = MapDriver.newMapDriver(mapper);   
25.            
26.         reduceDriver = ReduceDriver.newReduceDriver(reducer);   
27.         mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);   
28.     }   
29.        
30.     @Test  
31.     public void testMapper()   
32.     {   
33.         mapDriver.withInput(new LongWritable(), new Text(   
34.         "655209;1;796764372490213;804422938115889;6"));   
35.         mapDriver.withOutput(new Text("6"), new IntWritable(1));   
36.         mapDriver.runTest();   
37.     }   
38.        
39.     @Test  
40.     public void testReducer()   
41.     {   
42.         List<IntWritable> values = new ArrayList();   
43.         values.add(new IntWritable(1));   
44.         values.add(new IntWritable(1));   
45.         reduceDriver.withInput(new Text("6"), values);   
46.         reduceDriver.withOutput(new Text("6"), new IntWritable(2));   
47.         reduceDriver.runTest();   
48.     }   
49. }  

import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import flex.messaging.io.ArrayList;

public class SMSCDRMapperReducerTest
{
    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
    @Before
    public void setup()
    {
        SMSCDRMapper mapper = new SMSCDRMapper();
        SMSCDRReducer reducer = new SMSCDRReducer();
        mapDriver = MapDriver.newMapDriver(mapper);
        
        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
    }
    
    @Test
    public void testMapper()
    {
        mapDriver.withInput(new LongWritable(), new Text(
        "655209;1;796764372490213;804422938115889;6"));
        mapDriver.withOutput(new Text("6"), new IntWritable(1));
        mapDriver.runTest();
    }
    
    @Test
    public void testReducer()
    {
        List<IntWritable> values = new ArrayList();
        values.add(new IntWritable(1));
        values.add(new IntWritable(1));
        reduceDriver.withInput(new Text("6"), values);
        reduceDriver.withOutput(new Text("6"), new IntWritable(2));
        reduceDriver.runTest();
    }
}

 

Test测试类

 

 

Java代码  

1. import java.util.List;   
2.   
3. import org.apache.hadoop.io.IntWritable;   
4. import org.apache.hadoop.io.LongWritable;   
5. import org.apache.hadoop.io.Text;   
6. import org.apache.hadoop.mrunit.mapreduce.MapDriver;   
7. import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;   
8. import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;   
9. import org.junit.Before;   
10. import org.junit.Test;   
11.   
12. import flex.messaging.io.ArrayList;   
13.   
14. public class SMSCDRMapperReducerTest   
15. {   
16.     MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;   
17.     ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;   
18.     MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;   
19.     @Before  
20.     public void setup()   
21.     {   
22.         SMSCDRMapper mapper = new SMSCDRMapper();   
23.         SMSCDRReducer reducer = new SMSCDRReducer();   
24.         mapDriver = MapDriver.newMapDriver(mapper);   
25.            
26.         reduceDriver = ReduceDriver.newReduceDriver(reducer);   
27.         mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);   
28.     }   
29.        
30.     @Test  
31.     public void testMapper()   
32.     {   
33.         mapDriver.withInput(new LongWritable(), new Text(   
34.         "655209;1;796764372490213;804422938115889;6"));   
35.         mapDriver.withOutput(new Text("6"), new IntWritable(1));   
36.         mapDriver.runTest();   
37.     }   
38.        
39.     @Test  
40.     public void testReducer()   
41.     {   
42.         List<IntWritable> values = new ArrayList();   
43.         values.add(new IntWritable(1));   
44.         values.add(new IntWritable(1));   
45.         reduceDriver.withInput(new Text("6"), values);   
46.         reduceDriver.withOutput(new Text("6"), new IntWritable(2));   
47.         reduceDriver.runTest();   
48.     }   
49. }  

import java.util.List;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.junit.Before;
import org.junit.Test;

import flex.messaging.io.ArrayList;

public class SMSCDRMapperReducerTest
{
    MapDriver<LongWritable, Text, Text, IntWritable> mapDriver;
    ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
    MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
    @Before
    public void setup()
    {
        SMSCDRMapper mapper = new SMSCDRMapper();
        SMSCDRReducer reducer = new SMSCDRReducer();
        mapDriver = MapDriver.newMapDriver(mapper);
        
        reduceDriver = ReduceDriver.newReduceDriver(reducer);
        mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
    }
    
    @Test
    public void testMapper()
    {
        mapDriver.withInput(new LongWritable(), new Text(
        "655209;1;796764372490213;804422938115889;6"));
        mapDriver.withOutput(new Text("6"), new IntWritable(1));
        mapDriver.runTest();
    }
    
    @Test
    public void testReducer()
    {
        List<IntWritable> values = new ArrayList();
        values.add(new IntWritable(1));
        values.add(new IntWritable(1));
        reduceDriver.withInput(new Text("6"), values);
        reduceDriver.withOutput(new Text("6"), new IntWritable(2));
        reduceDriver.runTest();
    }
}

 

转自http://piaoling.iteye.com/blog/1672683