使用MRUnit测试hadoop的MapReduce

最新推荐文章于 2024-10-18 19:12:03 发布

wangqiaowqo

最新推荐文章于 2024-10-18 19:12:03 发布

阅读量128

点赞数

分类专栏： Hadoop 文章标签：大数据移动开发 jira

本文链接：https://blog.csdn.net/wangqiaowqo/article/details/84708427

版权

Hadoop 专栏收录该内容

15 篇文章 0 订阅

订阅专栏

1、参考
http://www.cnblogs.com/gpcuster/archive/2009/10/04/1577921.html
http://conkeyn.iteye.com/blog/2028437

2、实战


官方文档 wiki：https://cwiki.apache.org/confluence/display/MRUNIT/MRUnit+Tutorial
具体配置：
pom：
    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-core</artifactId>
            <version>1.2.1</version>
        </dependency>

        <dependency>
            <groupId>org.mockito</groupId>
            <artifactId>mockito-core</artifactId>
            <version>1.10.19</version>
        </dependency>

        <dependency>
            <groupId>org.apache.mrunit</groupId>
            <artifactId>mrunit</artifactId>
            <version>0.9.0-incubating</version>
            <classifier>hadoop1</classifier>
        </dependency>

        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.1.31</version>
        </dependency>

        <dependency>
            <groupId>com.laifeng</groupId>
            <artifactId>ddshow-hadoop</artifactId>
            <version>0.0.1-SNAPSHOT</version>
        </dependency>
    </dependencies>


public class LaifengClientInfoTest {

    MapDriver<LongWritable, Text, Text, Text> mapDriver;
    @Before
    public void setUp() {
        LaifengClientInfoAccessStat.ZMapper zMapper=new LaifengClientInfoAccessStat.ZMapper();
        mapDriver = MapDriver.newMapDriver(zMapper);

    }

    @Test
    public void testMapper() throws IOException {
        mapDriver.withInput(new LongWritable(1), new Text("0\t10.155.16.2\t1420356995734\t{\"clientInfo\":{\"appId\":\"1011\",\"appVersion\":\"1.1.0.0\",\"osVersion\":\"Windows_Win7\",\"deviceToken\":\"A4:1F:72:8A:2C:26\"}}\t{\"dataInfo\":{\"category\":\"plugin.sendSpeed\",\"room\":\"49\",\"cpuUsage\":\"17\",\"cpuAllusage\":\"31\",\"memoryUsage\":\"0\",\"memorySize\":\"146\",\"memoryAllusage\":\"75\",\"encoderMode\":\"0\",\"sendSpeed\":\"79.77392578125\",\"sendList\":\"0\",\"sendStatus\":\"100\",\"isReCon\":\"0\",\"reConTime\":\"0\",\"logtime\":\"1420356973734\",\"userAgent\":\"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; Tablet PC 2.0; InfoPath.3)\"}}"));
        // mapDriver.withOutput(new Text("6"), new IntWritable(1));
        //mapDriver.runTest();
        List<Pair<Text,Text>> list= mapDriver.run();
        for(Pair<Text,Text> p : list){
            System.out.println(p.getFirst()+"------"+p.getSecond());
        }


    }

}

------------------------------------------------
多条测试
https://issues.apache.org/jira/secure/attachment/12536974/MRUNIT-64.patch
http://my.oschina.net/cloudcoder/blog/285151


<dependency>
            <groupId>org.apache.mrunit</groupId>
            <artifactId>mrunit</artifactId>
            <version>1.1.0</version>
            <classifier>hadoop1</classifier>
        </dependency>
        <dependency>
            <groupId>org.mockito</groupId>
            <artifactId>mockito-all</artifactId>
            <version>1.9.5</version>
        </dependency>


final List<Pair<Object, Text>> inputs = new ArrayList<Pair<Object, Text>>();
        inputs.add(new Pair<Object, Text>(new LongWritable(1), new Text("0\t0\t0\t1428249610835\t42.243.188.250\t0\t0\t0\t0\t1670393750\t1428249610835\t\t0\t1670393750\t0\t0\t0\t0\tGET\t/anchor/search/sdk\tclientInfo=2003%7C3550325730_57%7C350%7C82678%7C0___%7C1%7Candroid_4.3%7C660bbfe4e50246ab6%7C0\t\tMozilla/5.0 (Linux; U; Android 4.3; zh-cn; HM 1SC Build/JLS36C) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30\t{\"ext\":{\"appId\":2003}}\t0\t0")));
        inputs.add(new Pair<Object, Text>(new LongWritable(1), new Text("0\t0\t0\t1428249610836\t42.243.188.250\t0\t0\t0\t0\t1670393750\t1428249610835\t\t0\t1670393750\t0\t0\t0\t0\tGET\t/anchor/search/sdk\tclientInfo=2003%7C3550325730_57%7C350%7C82678%7C0___%7C1%7Candroid_4.3%7C660bbfe4e50246ab6%7C0\t\tMozilla/5.0 (Linux; U; Android 4.3; zh-cn; HM 1SC Build/JLS36C) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30\t{\"ext\":{\"appId\":2003}}\t0\t0")));
        inputs.add(new Pair<Object, Text>(new LongWritable(1), new Text("0\t0\t0\t1428249610837\t42.243.188.250\t0\t0\t0\t0\t1670393750\t1428249610835\t\t0\t1670393750\t0\t0\t0\t0\tGET\t/anchor/search/sdk\tclientInfo=2003%7C3550325730_57%7C350%7C82678%7C0___%7C1%7Candroid_4.3%7C660bbfe4e50246ab6%7C0\t\tMozilla/5.0 (Linux; U; Android 4.3; zh-cn; HM 1SC Build/JLS36C) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30\t{\"ext\":{\"appId\":2003}}\t0\t0")));

        mapDriver.withAll(inputs);

        List<Pair<Text,Text>> list= mapDriver.run();
        for(Pair<Text,Text> p : list){
            System.out.println(p.getFirst()+"##end##"+p.getSecond());
        }