WordCount经典编程

WordCount原理：
MapReduce任务被初始化为一个Job，每个Job又可以分为两种阶段：map阶段和reduce阶段。这两个阶段分别用两个函数表示，即map函数和reduce函数。

/**
 * WordCount.java
 * com.hainiu.hadoop.mr
 * Copyright (c) 2017, 小马同学版权所有.
 * @author   小马同学
*/

package com.xiaoma.hadoop.mr;

import java.io.IOException;

import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;

/**
 *
 * @author   小马同学
 * @Date     2017年5月16日      
 */
public class WordCount extends Configured implements Tool {
    //内部类Map
    //com.hainiu.hadoop.mr.WordCount
    public static class WordCountMapper extends Mapper<LongWritable, Text, Text, LongWritable> {
        private Text word = new Text();
        private LongWritable one = new LongWritable(1);

        //重新实现增加相应的功能
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            //所有原始数据
            String str = value.toString();
            //定义一个数组元素是按空格分割
            String strs[] = str.split(" ");
            //遍历
            for (String s : strs) {
                //把单词元素s设置进去
                word.set(s);
                //把单词及其统计写出来   型如： Hello 1 1 1 1 
                context.write(word, one);
            }

        }

    }

    //内部类Reduce
    public static class WordCountReducer extends Reducer<Text, LongWritable, Text, LongWritable> {
        private LongWritable valueout = new LongWritable();

        @Override
        protected void reduce(Text key, Iterable<LongWritable> values, Context context)
                throws IOException, InterruptedException {
            //用于统计每个单词的总个数
            long sum = 0L;
            //统计单词个数
            for (LongWritable num : values) {
                sum += num.get();
            }
            //将个数sum设置进去
            valueout.set(sum);
            context.write(key, valueout);

        }

    }

    public int run(String[] args) throws Exception {
                if (args.length !=2){
                    System.out.println("必须输入两个参数，<input> <output>");
                    return 1;
                }
        //定义job名字并得到配置
        Job job = Job.getInstance(getConf(), "wordcount");
        //设置jar使用的class
        job.setJarByClass(WordCount.class);
        //设置使用的mapper类型
        job.setMapperClass(WordCountMapper.class);
        //设置使用的reduce类型
        job.setReducerClass(WordCountReducer.class);
        //设置输出的key类型
        job.setOutputKeyClass(Text.class);
        //设置输出的value类型
        job.setOutputValueClass(LongWritable.class);
        //设置任务的输入地址
        FileInputFormat.addInputPath(job, new Path(args[0]));
        //设置任务的输出地址
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        //删除以前存在的目录

        FileSystem fs = FileSystem.get(getConf());
        if(fs.exists(new Path(args[1]))){
            fs.delete(new Path(args[1]), true);
            System.out.println("out put delete finish");
        }
        //提交状态
        return job.waitForCompletion(true) ? 0 : 1;
    }

    public static void main(String[] args) throws Exception {
        int res = ToolRunner.run(new WordCount(), args);
        System.exit(res);

    }
}