Hadoop练习wordcout+多文件输出

package xxx.hadoop;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import xxx.bigdata.common.ConfigurationHadoop;

public class MultipleOutPut {

public static class TokenizerMapper extends
Mapper<Object, Text, Text, IntWritable> {

private final static IntWritable one = new IntWritable(1);
private Text word = new Text();

public void map(Object key, Text value, Context context)
throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
context.write(word, one);
}
}
}

public static class IntSumReducer extends
Reducer<Text, IntWritable, Text, IntWritable> {

private static MultipleOutputs<Text, IntWritable> output;

protected void setup(Context context) throws IOException,
InterruptedException {
output = new MultipleOutputs<Text, IntWritable>(context);
}

private IntWritable result = new IntWritable();

public void reduce(Text key, Iterable<IntWritable> values,
Context context) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
if (sum == 1) {
output.write(key, result, "output-1");
} else {
output.write(key, result, "output-2");
}
}

@Override
protected void cleanup(Context context) throws IOException,
InterruptedException {
super.cleanup(context);
output.close();
}
}

public static void main(String[] args) throws Exception {
// Configuration conf = new Configuration();
Configuration conf = ConfigurationHadoop.getConfigurationHadoop();

String[] otherArgs = new String[] { "/user/hdfs/input", "/user/hdfs/output" };

Job job = Job.getInstance(conf, "MultipleOutPut");
job.setJarByClass(MultipleOutPut.class);

/*
    关掉 speculative execution功能。
speculative execution功能是指，假如Hadoop发现有些任务执行的比较慢，那么，它会在其他的节点上再运行一个同样的任务。这两个任务，哪个先完成就以哪个结果为准。
    但Reduce任务需要将数值写入到HDFS的文件里，而且这个文件名是固定的，如果同时运行两个以上的Reduce任务，会导致写入出错，所以要关闭这个功能。
*/
job.setSpeculativeExecution(false);
job.setMapSpeculativeExecution(false);
job.setReduceSpeculativeExecution(false);

job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);

job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
// job.setOutputFormatClass(TextOutputFormat.class);
LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);

FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
FileSystem hdfs = FileSystem.get(conf);
if (hdfs.exists(new Path(otherArgs[1]))) {
hdfs.delete(new Path(otherArgs[1]), true);
}
FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));

System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}