mapreduce代码整理之wordcount

大家都说wordcount是mapreduce中的halloword，代码如下。

import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;  
import java.util.StringTokenizer;  

import org.apache.hadoop.conf.Configuration;  
import org.apache.hadoop.fs.Path;  
import org.apache.hadoop.io.IntWritable;  
import org.apache.hadoop.io.LongWritable;  
import org.apache.hadoop.io.Text;  
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapreduce.Job;  
import org.apache.hadoop.mapreduce.Mapper;  
import org.apache.hadoop.mapreduce.Reducer;  
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;  
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
public class fenxi{
public static class WordCountMap extends Mapper<LongWritable, Text, Text, IntWritable> {  

private final IntWritable one = new IntWritable(1);  
private Text word = new Text();  

public void map(LongWritable key, Text value,Context context) throws IOException, InterruptedException {  
    String line = value.toString();  
    StringTokenizer token = new StringTokenizer(line);  
    while (token.hasMoreTokens()) {  
        word.set(token.nextToken()); 
        //System.out.println(word.toString()+one.toString());
        context.write(word, one);
    }  
}  
}  

public static class WordCountReduce extends  Reducer<Text, IntWritable, Text, IntWritable> {  

public void reduce(Text key, Iterable<IntWritable> values,Context context) throws IOException, InterruptedException {  
    int sum = 0;  
    for (IntWritable val : values) {  
        sum += val.get();  
    }  
    context.write(key, new IntWritable(sum)); 
    context.write(key, new IntWritable(sum));
}  
}  

public static void main(String[] args) throws Exception {  
Configuration conf = new Configuration();
conf.setStrings("mapred.tasktracker.reduce.tasks.maximum", "4");
 //conf.setInt("mapred.tasktracker.reduce.tasks.maximum", 2);
//conf.setCompressMapOutput(true);  
//conf.setMapOutputCompressorClass(GzipCodec.class); 
//conf.setBoolean("mapred.compress.map.output", true);//代码实现map输出压缩减少网络传输压力
//conf.setClass("mapred.map.output.compression.codec", GzipCodec.class, CompressionCodec.class);
Job job = new Job(conf);  
job.setJarByClass(fenxi.class);  
job.setJobName("wordcount");  

job.setOutputKeyClass(Text.class);  
job.setOutputValueClass(IntWritable.class);  
job.setNumReduceTasks(2);
job.setMapperClass(WordCountMap.class);  
job.setReducerClass(WordCountReduce.class);  

job.setInputFormatClass(TextInputFormat.class);  
job.setOutputFormatClass(TextOutputFormat.class);  

FileInputFormat.addInputPath(job, new Path(args[0]));  
FileOutputFormat.setOutputPath(job, new Path(args[1]));  

//FileOutputFormat.setCompressOutput(job,true);//代码实现输出压缩
//FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);

job.waitForCompletion(true);  
}  
}