MapReduce 之Counter

最新推荐文章于 2021-02-26 05:38:18 发布

???Sir

最新推荐文章于 2021-02-26 05:38:18 发布

阅读量117

点赞数

文章标签：大数据 python

原文链接：https://my.oschina.net/momisabuilder/blog/533619

版权

2019独角兽企业重金招聘Python工程师标准>>>

一、Main代码大放送

        JobConf job = new JobConf(new Configuration(), HadoopWordCount.class);
        job.setJobName("word-count-job");
        //设置Mapper的输出的key和value的类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        //输入和输出格式化
        job.setInputFormat(TextInputFormat.class);
        job.setOutputFormat(TextOutputFormat.class);
        job.setMapperClass(MyMapper.class);
        job.setReducerClass(MyReducer.class);
        //这是整个任务的的输出key、value
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        //格式化路径
        FileInputFormat.setInputPaths(job, new Path(this.inputPath));
        FileOutputFormat.setOutputPath(job, new Path("/lwj/" + new Random().nextInt() + "wordcount.txt"));
        job.setNumReduceTasks(1);
        JobClient client = new JobClient(jobConf);
        RunningJob runningJob = client.runJob(jobConf);
        if (runningJob.isComplete()) {
            Counters counters = runningJob.getCounters();
            LOG.info("Map Input Record : " + counters.getCounter(Task.Counter.MAP_INPUT_RECORDS));
            LOG.info("Map Output Record : " + counters.getCounter(Task.Counter.MAP_OUTPUT_RECORDS));
            LOG.info("Reduce Input Record : " + counters.getCounter(Task.Counter.REDUCE_INPUT_RECORDS));
            LOG.info("Reduce Output Record : " + counters.getCounter(Task.Counter.REDUCE_OUTPUT_RECORDS));
            LOG.info("Combine Input Record : " + counters.getCounter(Task.Counter.COMBINE_INPUT_RECORDS));
            LOG.info("Combine Output Record : " + counters.getCounter(Task.Counter.COMBINE_OUTPUT_RECORDS));

           Counters.Counter totalNumber = counters.findCounter("totalNumber", MyMapper.MyCounters.NUM_RECORDS.toString());
           LOG.info("================" + totalNumber.getValue());
        }

二、Map任务

public static class MyMapper implements Mapper<LongWritable, Text, Text, IntWritable> {
        static enum MyCounters {NUM_RECORDS}

        @Override
        public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
            //reporter.progress();
            //reporter.incrCounter(MyCounters.NUM_RECORDS, 1);
            reporter.incrCounter("totalNumber", MyCounters.NUM_RECORDS.name(), 1);
        }
    }

转载于:https://my.oschina.net/momisabuilder/blog/533619