- 加入MRUnit依赖
<dependency> <groupId>com.cloudera.hadoop</groupId> <artifactId>hadoop-mrunit</artifactId> <version>0.20.2-320</version> <scope>test</scope> </dependency>
- 单独测试Map
public class WordCountMapperTest { private Mapper mapper; private MapDriver driver; @Before public void init(){ mapper = new WordCountMapper(); driver = new MapDriver(mapper); } @Test public void test() throws IOException{ String line = "Taobao is a great website"; driver.withInput(null,new Text(line)) .withOutput(new Text("Taobao"),new IntWritable(1)) .withOutput(new Text("is"), new IntWritable(1)) .withOutput(new Text("a"), new IntWritable(1)) .withOutput(new Text("great"), new IntWritable(1)) .withOutput(new Text("website"), new IntWritable(1)) .runTest(); } }
- 单独测试Reduce
public class WordCountReducerTest { private Reducer reducer; private ReduceDriver driver; @Before public void init(){ reducer = new WordCountReducer(); driver = new ReduceDriver(reducer); } @Test public void test() throws IOException{ String key = "taobao"; List values = new ArrayList(); values.add(new IntWritable(2)); values.add(new IntWritable(3)); driver.withInput(new Text("taobao"), values) .withOutput(new Text("taobao"), new IntWritable(5)) .runTest(); } }
- 测试MapReduce
public class WordCountTest { private Mapper mapper; private Reducer reducer; private MapReduceDriver driver; @Before public void init(){ mapper = new WordCountMapper(); reducer = new WordCountReducer(); driver = new MapReduceDriver(mapper,reducer); } @Test public void test() throws RuntimeException, IOException{ String line = "Taobao is a great website, is it not?"; driver.withInput("",new Text(line)) .withOutput(new Text("Taobao"),new IntWritable(1)) .withOutput(new Text("a"),new IntWritable(1)) .withOutput(new Text("great"),new IntWritable(1)) .withOutput(new Text("is"),new IntWritable(2)) .withOutput(new Text("it"),new IntWritable(1)) .withOutput(new Text("not"),new IntWritable(1)) .withOutput(new Text("website"),new IntWritable(1)) .runTest(); } }
- 重构后的测试代码
@Test public void test() throws RuntimeException, IOException{ String line = "Taobao is a great website, is it not?"; List<Pair> out = null; out = driver.withInput("",new Text(line)).run(); List<Pair> expected = new ArrayList<Pair>(); expected.add(new Pair(new Text("Taobao"),new IntWritable(1))); expected.add(new Pair(new Text("a"),new IntWritable(1))); expected.add(new Pair(new Text("great"),new IntWritable(1))); expected.add(new Pair(new Text("is"),new IntWritable(2))); expected.add(new Pair(new Text("it"),new IntWritable(1))); expected.add(new Pair(new Text("not"),new IntWritable(1))); expected.add(new Pair(new Text("website"),new IntWritable(1))); assertListEquals(expected, out); }
Hadoop学习笔记之三:用MRUnit做单元测试
引言
借
年底盛宴品鉴
之风,继续抒我Hadoop之情,本篇文章介绍如何对Hadoop的MapReduce进行单元测试。MapReduce的开发周期差不多是这样:编写mapper和reducer、编译、打包、提交作业和结果检索等,这个过程比较繁琐,一旦提交到分布式环境出了问题要定位调试,重复这样的过程实在无趣,因此先对MapReduce做单元测试,消除明显的代码bug尤为必要。
MRUnit简介
MRUnit是一款由Couldera公司开发的专门针对Hadoop中编写MapReduce单元测试的框架。可以用MapDriver单独测试Map,用ReduceDriver单独测试Reduce,用MapReduceDriver测试MapReduce作业。
实战
我们将利用MRUnit对本系列上篇文章
MapReduce基本编程
中的字数统计功能进行单元测试。