wordCount MapReduce

map

package os.unix.cn;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

/*
 * KEYIN:输入数据KV对中的key数据类型 (行的起始偏移量)
 * VALUEIN:输入数据KV队中的value数据类型
 * KEYOUT: 输出KV对数据中的数据类型
 * VALUEOUT: 输出数据KV中的数据类型
 */
public class WordCountMapper extends Mapper<LongWritable,Text,Text,LongWritable>{

    /* map方法是提供给maptask 进程来调用的,map task进程是每读取一行来调用一次我们自己定义的map方法
     * map task 在调用map方法的时候,传递的参数 是,一行的起始偏移量 LongWritable作为key,一行的文本内容Text作为value
     */

    @Override
    protected void map(LongWritable key, Text value, Context context)throws IOException, InterruptedException {

        // 拿到一行文本内容转换到string类型
        String line = value.toString();

        //将这一行文本切成 单词
        String[] words = line.split(" ");

        // 把单词输出 <key,1>
        for(String word:words) {
            context.write(new Text(word), new LongWritable(1));
        }
    }


}

reduce

package os.unix.cn;

import java.io.IOException;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;


/*
 * KEYIN:对于map阶段输出的key类型
 * VALUE:对应map阶段输出的value类型
 * KEYOUT:对应reduce处理完之后输出的结果KV对key类型
 * VALUEOUT:对应reduce处理完输出结果value类型
 */
public class WordCountReducer extends Reducer<Text, LongWritable, Text, LongWritable>{

    /*
     * reduce 方法是提供给reduce task进程调用
     * reduce task会将shuffle阶段分发过来的KV对进行聚合,聚合的依据是相同key的kv对聚合为一组
     * 然后reduce task对每一组聚合kv调用一次我们自定义的reduce方法
     * @see org.apache.hadoop.mapreduce.Reducer#reduce(KEYIN, java.lang.Iterable, org.apache.hadoop.mapreduce.Reducer.Context)
     */
    @Override
    protected void reduce(Text key, Iterable<LongWritable> values,Context context) throws IOException, InterruptedException {

        //定义一个计数器
        long count = 0;

        //通过values 这个迭代器,遍历一组kv中所有的value,进行累加
        for(LongWritable value: values) {
            count += value.get();
        }

        //输出这个单词的统计结果
        context.write(key, new LongWritable(count));
    }
}

---

main

package os.unix.cn;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class WordCountJob {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException{

        Configuration conf = new Configuration();
        Job wordCountJob = Job.getInstance(conf);

        // 指定本Job的 所在的jar包
        wordCountJob.setJarByClass(WordCountJob.class);

        //设置wordCountJob 所用的mapper逻辑类为那个类
        wordCountJob.setMapperClass(WordCountMapper.class);

        //设置wordCountJob 所用的mapper逻辑类为那个类
        wordCountJob.setReducerClass(WordCountReducer.class);

        // map阶段输出的kv类型
        wordCountJob.setMapOutputKeyClass(Text.class);
        wordCountJob.setMapOutputValueClass(LongWritable.class);


        // reduce阶段输出的kv类型
        wordCountJob.setOutputKeyClass(Text.class);
        wordCountJob.setOutputValueClass(LongWritable.class);

        //设置的要处理的文本数据存放路径
        FileInputFormat.setInputPaths(wordCountJob, "hdfs://os-1:9000/word/srcdata/");

        //设置最终结果所存放的路径
        FileOutputFormat.setOutputPath(wordCountJob, new Path("hdfs://os-1:9000/wordcount/output/"));


        wordCountJob.waitForCompletion(true);
    }
}

打包之后 
执行 hadoop jar wordCount.jar os.unix.cn.WordCountJob