使用MRUnit和TestNG进行单元测试

 

MRUnit

　　MRUnit是一款由Couldera公司开发的专门针对Hadoop中编写MapReduce单元测试的框架。

 

定义Map逻辑

import java.io.IOException;

import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Mapper;


public class WordMapper extends Mapper<LongWritable, Text, Text, Text> 
{
    @Override
    public void map(LongWritable key, Text value, Context context) throws InterruptedException, IOException 
    {
        String[] line = value.toString().split(" ");
        context.write(new Text(line[0]), new Text(line[1]));
    }
}

 

定义Reduce逻辑

import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;


public class WordReducer extends Reducer<Text, Text, Text, Text> 
{
    public void reduce(Text key, Iterable<Text> values, Context context) throws InterruptedException, IOException 
    {
        for (Text value : values) {
            context.write(value, key);
        }
    }
}

 

单元测试示例

import java.util.*;

import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.mrunit.mapreduce.*;
import org.apache.hadoop.mrunit.types.Pair;
import org.testng.annotations.*;


public class UserForAreaDistributeTest 
{
    private Mapper<LongWritable, Text, Text, Text> mapper;
    private MapDriver<LongWritable, Text, Text, Text> mapDriver;
    
    private Reducer<Text, Text, Text, Text> reducer;
    private ReduceDriver<Text, Text, Text, Text> reduceDriver;
    
    
    @BeforeClass
    public void setUp()
    {
        mapper = new WordMapper();
        mapDriver = new MapDriver(mapper);
        
        reducer = new WordReducer();
        reduceDriver = new ReduceDriver(reducer);
    }
    
    @Test
    public void testMap() 
    {
        mapDriver.withInput(new Pair(new LongWritable(1L), new Text("abc def")));
        mapDriver.withOutput(new Text("abc"), new Text("def"));
        mapDriver.runTest();
    }
    
    @Test
    public void testReduce()
    {
        List<Text> list = new ArrayList<Text>();
        list.add(new Text("value1"));
        list.add(new Text("value2"));
        
        reduceDriver.withInput(new Text("key"), list);
        reduceDriver.withOutput(new Text("value1"), new Text("key"));
        reduceDriver.withOutput(new Text("value2"), new Text("key"));
        reduceDriver.runTest();
    }
}

转载于:https://www.cnblogs.com/rilley/archive/2012/11/30/2795780.html