Java中使用MapReduce实现WordCount

Java中使用MapReduce实现WordCount

Map端

import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;

public class WordCountMap extends Mapper<LongWritable, Text, Text, IntWritable>
{
	//定义一个输出key
    Text outPutKey = new Text();

	@Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException
    {
    	//得到一行后，按空格分割
        String[] words = value.toString().split("\\s+");
        for (String word : words)
        {
            outPutKey.set(word)；
            context.write(outPutKey, new IntWritable(1));
        }
    }
}

Combiner端（可选）

Combiner端和Reduce端代码一样，其设定是为了减少传输到Reduce中的数据量，
减少网络带宽和Reduce的负载

Reduce端

import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
import java.util.Iterator;

public class WordCountReduce extends Reducer<Text, IntWritable, Text, IntWritable>
{
	//定义一个输出值
    IntWritable outPutValue = new IntWritable();

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException
    {
        int sum = 0;
        Iterator<IntWritable> iter = values.iterator();
        while (iter.hasNext())
        {
            IntWritable num = iter.next();
            sum += num.get();
        }
        outPutValue.set(sum);
        context.write(key,outPutValue);
    }
}

Driver端

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;

public class WordCountDriver
{
    public static void main( String[] args ) throws IOException, ClassNotFoundException, InterruptedException
    {
        //创建配置文件
        Configuration conf = new Configuration();

        //创建job
        Job job = Job.getInstance(conf);

        //设置class
        job.setJarByClass(WordCountDriver.class);
        job.setMapperClass(WordCountMap.class);
        job.setReducerClass(WordCountReduce.class);
        job.setCombinerClass(WordCountCombiner.class);

        //设置job名称
        job.setJobName("wordcount");

        //设置Map输入输出的 key，value
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
		
		设置Reduce输入输出的 key，value
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        //设置输入输出路径
        FileInputFormat.addInputPath(job,new Path("/test/wc.txt"));

        /*
        	检查是否存在输出路径
        	因为路径存在就会报错，搜索先检查一下
        */
        FileSystem fs=FileSystem.get(conf);
        Path path = new Path("/test/output");
        if(fs.exists(path))
        {
            fs.delete(path,true);
        }
        fs.close();
        FileOutputFormat.setOutputPath(job, path);

        //提交job
        boolean result=job.waitForCompletion(true);
        System.exit(result?0:1);
    }
}

备注

1. 文件只需随意写一个文件就行，但是需要以空格作为分割，如：

2. 若Driver端的configuration没有特别设置，则需要提前将配置文件拷贝到resources中