//MapReduce 排序例子 import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.Reducer; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import java.io.IOException; public class Sort { //12,3,4,5,6, /* 1 3 2 4*/ // map key public static class SortMap extends Mapper<Object,Text,IntWritable,IntWritable>{ // private static IntWritable data=new IntWritable(); @Override protected void map(Object key,Text values,Context context) throws IOException, InterruptedException { // String line=values.toString(); data.set(Integer.parseInt(line)); context.write(data,new IntWritable(1)); } } public static class SortReduce extends Reducer<IntWritable,IntWritable,IntWritable,IntWritable>{ private static IntWritable linenum=new IntWritable(1); @Override protected void reduce(IntWritable key,Iterable<IntWritable> values,Context context) throws IOException, InterruptedException { for(IntWritable value:values){ context.write(linenum,key); linenum= new IntWritable(value.get()+1); } } } public static int main(String[] args) throws Exception{ Configuration conf =new Configuration(); Job job=Job.getInstance(conf); job.setJarByClass(Sort.class); job.setMapperClass(SortMap.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); // job.setReducerClass(SortReduce.class); job.setOutputKeyClass(IntWritable.class); job.setMapOutputValueClass(IntWritable.class); // Path inPath =new Path(args[0]); FileInputFormat.addInputPath(job,inPath); Path outPath=new Path(args[1]); FileSystem fs =FileSystem.get(conf); if(fs.exists(outPath)){ fs.delete(outPath,true); } FileOutputFormat.setOutputPath(job,outPath); boolean b = job.waitForCompletion(true); return b?1:0; } }
MapReduce例子
最新推荐文章于 2022-06-26 12:15:07 发布