2.MapReduce排序

最新推荐文章于 2024-10-15 20:04:37 发布

花开的那一天

最新推荐文章于 2024-10-15 20:04:37 发布

阅读量365

点赞数 11

文章标签： mapreduce

本文链接：https://blog.csdn.net/xyzzzH/article/details/135238984

版权

1.源文件获取数据

2.map将读到的每行数据，一行行写给reduce

public class SortMapper extends Mapper<LongWritable, Text, Text, NullWritable> {

    @Override
    protected void map(LongWritable key, Text value, Mapper<LongWritable, Text, Text, NullWritable>.Context context) throws IOException, InterruptedException {
        context.write(value, NullWritable.get());
    }
}

3.设置排序规则

public class SortComparator extends WritableComparator {
    public SortComparator() {
        super(Text.class, true);
    }

    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        Text left = (Text)a;
        Text right = (Text)b;
        return right.compareTo(left);
    }
}

4.reduce将数据写出到文件

public class SortReducer extends Reducer<Text, NullWritable, Text, NullWritable> {
    @Override
    protected void reduce(Text key, Iterable<NullWritable> values, Reducer<Text, NullWritable, Text, NullWritable>.Context context) throws IOException, InterruptedException {
        //        相同的key 不去重
//        for (NullWritable value : values) {
//            context.write(key, NullWritable.get());
//        }
        context.write(key, NullWritable.get());
    }
}

5.SortLaunch驱动

public class SortLaunch {
    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        BasicConfigurator.configure();
        Job job = Job.getInstance();
        job.setJobName("sort");
        job.setJarByClass(SortLaunch.class);

        job.setMapperClass(SortMapper.class);
        job.setReducerClass(SortReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        job.setSortComparatorClass(SortComparator.class);

        FileInputFormat.addInputPath(job, new Path("D:\\stu\\code\\hadoop_project\\sort1\\input"));

        Path out = new Path("D:\\stu\\code\\hadoop_project\\sort1\\output");

        // 获取文件管理目录
        FileSystem fs = FileSystem.get(job.getConfiguration());

        if (fs.exists(out)){
            fs.delete(out, true);
        }

        FileOutputFormat.setOutputPath(job, out);

        // 设置 Reduce 任务数量
        // 如果要对整个数据集进行全局排序，可以将 Reduce 任务的数量设置为 1
        job.setNumReduceTasks(1);
        // 提交
        job.waitForCompletion(true);
    }
}