hadoop的数据倾斜之自定义分区解决（记录七----4）

https://mp.csdn.net/postedit/86480018(hadoop的数据倾斜之自定义分区解决（记录七----4）)

//主函数
public class SkewApp1 {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration configuration=new Configuration();
        configuration.set("fs.defaultFS","file:///");
        Job job= Job.getInstance(configuration);

        job.setJobName("SkewApp");
        job.setJarByClass(SkewApp1.class);
        job.setInputFormatClass(TextInputFormat.class);

        FileInputFormat.addInputPath(job,new Path("d:/MR/skew/out2/part-r-00000"));
        FileInputFormat.addInputPath(job,new Path("d:/MR/skew/out2/part-r-00001"));
        FileInputFormat.addInputPath(job,new Path("d:/MR/skew/out2/part-r-00002"));
        FileInputFormat.addInputPath(job,new Path("d:/MR/skew/out2/part-r-00003"));
        FileOutputFormat.setOutputPath(job,new Path("d:/MR/skew/out1"));


        job.setMapperClass(SloveSkewMapper1.class);
        job.setReducerClass(SkewReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setNumReduceTasks(4);

        job.waitForCompletion(true);
    }
}
//map函数
class SloveSkewMapper1 extends Mapper<LongWritable,Text,Text,IntWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] arr=value.toString().split("\t");
        context.write(new Text(arr[0]),new IntWritable(Integer.parseInt(arr[1])));

    }
}
//reduce函数
public class SkewReducer1 extends Reducer<Text,IntWritable,Text,IntWritable> {
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int count=0;
        for(IntWritable iw:values){
            count=count+iw.get();
        }
        context.write(key,new IntWritable(count));
    }
}

结果<看“a”的数量>（可以发现“a”的记录二的一样，虽然结果一样，但是过程截然不同，记录二中的结果是把“a”的数量堆积在一个reduce上聚合，而现在这个结果，是先用随机分区，把“a”分散到不同节点进行初步聚合，然后在使用一个MapReduce框架程序，将不同的reduce节点上的“a”的初步结果，再一次进行聚合。）

part-r-00000
a	45
b1	1
b10	1
b14	1
b18	1
b21	1
b25	1
b29	1
b32	1
b36	1
b43	1
b5	1
b9	1

part-r-00001
b11	1
b15	1
b19	1
b2	1
b22	1
b26	1
b33	1
b37	1
b40	1
b44	1
b6	1

part-r-00002
b12	1
b16	1
b23	1
b27	1
b3	1
b30	1
b34	1
b38	1
b41	1
b45	1
b7	1

part-r-00003
b12	1
b16	1
b23	1
b27	1
b3	1
b30	1
b34	1
b38	1
b41	1
b45	1
b7	1