java reduce.mdn_MapReduce_去重抽取mdn_imsi

importjava.io.IOException;importorg.apache.hadoop.conf.Configuration;importorg.apache.hadoop.fs.Path;importorg.apache.hadoop.io.IntWritable;importorg.apache.hadoop.io.Text;importorg.apache.hadoop.mapreduce.Job;importorg.apache.hadoop.mapreduce.Mapper;importorg.apache.hadoop.mapreduce.Reducer;importorg.apache.hadoop.mapreduce.Reducer.Context;importorg.apache.hadoop.mapreduce.lib.input.FileInputFormat;importorg.apache.hadoop.mapreduce.lib.output.FileOutputFormat;importorg.apache.hadoop.util.GenericOptionsParser;public classMdnWithImsiTest {public static class MyMapper extends Mapper{private Text keytText = newText();private Text valuetText = newText();//private static Text line = new Text();

public void map(Object key, Text value, Context context)throwsIOException, InterruptedException {//TODO Auto-generated method stub

String[] items= value.toString().split("\\|");

System.out.println(items.length);//context.write(text.set(items[8]), text.set(items[1]));

keytText.set(items[8]);

valuetText.set(items[1]);

context.write(keytText, valuetText);//line=value;//context.write(line, new Text(""));

}

}public static class MyReducer extends Reducer{public void reduce(Text key, Iterable values, Context context) throwsIOException, InterruptedException {

Text new_value= newText();for(Text val2 : values) {

new_value=val2;

}

context.write(key, new_value);

}

}public static void main(String[] args) throwsException {

Configuration conf= newConfiguration();

String[] otherArgs= newGenericOptionsParser(conf, args)

.getRemainingArgs();

System.out.println(otherArgs.length);if (otherArgs.length != 2) {

System.err.println("Usage: wordcount ");

System.exit(2);

}

Job job= new Job(conf, "MdnImsi");

job.setJobName("MdnImsi");

job.setJarByClass(MdnWithImsiTest.class);

job.setMapperClass(MyMapper.class);

job.setCombinerClass(MyReducer.class);

job.setReducerClass(MyReducer.class);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(Text.class);

FileInputFormat.addInputPath(job,new Path(otherArgs[0]));

FileOutputFormat.setOutputPath(job,new Path(otherArgs[1]));

System.exit(job.waitForCompletion(true) ? 0 : 1);

}

}