hadoop的数据倾斜之自定义分区解决（记录七----2）

https://mp.csdn.net/postedit/86479388(hadoop的数据倾斜之自定义分区解决（记录七----1）)

package com.gsd.skew;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


import java.io.IOException;

/**
 * 数据倾斜
 */
public class SkewApp {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Configuration configuration=new Configuration();
        configuration.set("fs.defaultFS","file:///");
        Job job= Job.getInstance(configuration);

        job.setJobName("SkewApp");
        job.setJarByClass(SkewApp.class);
        job.setInputFormatClass(TextInputFormat.class);

        FileInputFormat.addInputPath(job,new Path("d:/MR/skew/*"));

        FileOutputFormat.setOutputPath(job,new Path("d:/MR/skew/out1"));

        job.setNumReduceTasks(4);
        job.setMapperClass(SkewMapper.class);
        job.setReducerClass(SkewReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.waitForCompletion(true);
    }
}

运行程序出现数据倾斜（结果如下）

part-r-00000       
a	45              
b1	1           
b10	1           
b14	1           
b18	1           
b21	1           
b25	1           
b29	1           
b32	1           
b36	1           
b43	1           
b5	1           
b9	1           

part-r-00001
b11	1
b15	1
b19	1
b2	1
b22	1
b26	1
b33	1
b37	1
b40	1
b44	1
b6	1

part-r-00002
b12	1
b16	1
b23	1
b27	1
b3	1
b30	1
b34	1
b38	1
b41	1
b45	1
b7	1

part-r-00003
b13	1
b17	1
b20	1
b24	1
b28	1
b31	1
b35	1
b39	1
b4	1
b42	1
b8	1