Hadoop MapReduce排序程序

    package com.hadoop.sample;  
      
    import java.io.IOException;  
    import java.util.StringTokenizer;  
      
    import org.apache.hadoop.conf.Configuration;  
    import org.apache.hadoop.fs.Path;  
    import org.apache.hadoop.io.IntWritable;  
    import org.apache.hadoop.io.Text;  
    import org.apache.hadoop.mapreduce.Job;  
    import org.apache.hadoop.mapreduce.Mapper;  
    import org.apache.hadoop.mapreduce.Reducer;  
    import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;  
    import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;  
    import org.apache.hadoop.util.GenericOptionsParser;  
      
    public class Sort {  
        //map将输入中的value化成IntWritable类型,作为输出的key  
        public static class Map extends Mapper<Object,Text,IntWritable,IntWritable>{  
            private static IntWritable data = new IntWritable();  
            public void map(Object key,Text value,Context context) throws IOException,InterruptedException{  
                 String line = value.toString();  
                 data.set(Integer.parseInt(line));  
                 context.write(data, new IntWritable(1));  
            }  
        }  
        //reduce将输入中的key复制到输出的value上,然后根据输入的  
        //value-list中的元素的个数决定key的输出次数  
        //用全局的linenum来代表key的位次  
        public static class Reduce extends Reducer<IntWritable,IntWritable,IntWritable,IntWritable>{  
            private static IntWritable linenum = new IntWritable(1);  
            public void reduce(IntWritable key,Iterable<IntWritable> values,Context context) throws IOException,InterruptedException{  
                for(IntWritable val:values){  
                    context.write(linenum, key);  
                    linenum = new IntWritable(linenum.get()+1);  
                }  
                  
            }  
        }  
        /** 
         * @param args 
         */  
        public static void main(String[] args) throws Exception{  
            // TODO Auto-generated method stub  
            Configuration conf = new Configuration();  
            String[] otherArgs = new GenericOptionsParser(conf,args).getRemainingArgs();  
            if(otherArgs.length != 2){  
                System.err.println("Usage WordCount <int> <out>");  
                System.exit(2);  
            }  
            Job job = new Job(conf,"Sort");  
            job.setJarByClass(Sort.class);  
            job.setMapperClass(Map.class);  
            job.setCombinerClass(Reduce.class);  
            job.setReducerClass(Reduce.class);  
            job.setOutputKeyClass(IntWritable.class);  
            job.setOutputValueClass(IntWritable.class);  
            FileInputFormat.addInputPath(job, new Path(otherArgs[0]));  
            FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));  
            System.exit(job.waitForCompletion(true) ? 0 : 1);  
        }  
      
    }  

  • 0
    点赞
  • 1
    收藏
    觉得还不错? 一键收藏
  • 1
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值