MapReduce之week2 test 分区计算结余(练习)

static {
System.setProperty(“hadoop.home.dir”,“E:/x3/hadoop-2.9.2”);
}

//map
public static class MyMapper extends Mapper<LongWritable,Text,Text,Week2Data>{
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String lineValue = value.toString();
        String[] split = lineValue.split("\t");
        String year = split[0].substring(0, 4);
        Integer diff = null;
        Integer num1 = Integer.parseInt(split[1]);
        Integer num2 = Integer.parseInt(split[2]);
        diff = num1 - num2;
        context.write(new Text(year),new Week2Data(split[0],diff));
    }
}

//自定义分区器
public static class MyPartitioner extends Partitioner<Text,Week2Data>{

    @Override
    public int getPartition(Text key, Week2Data value, int i) {
        return key.hashCode() % 127 % i;
    }
}

//自定义分组
public static class MyGroupComparble extends WritableComparator{
    protected MyGroupComparble() {
        super(Week2Data.class,true);
    }

    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        Week2Data data1 = (Week2Data) a;
        Week2Data data2 = (Week2Data) b;

        return data1.getYear().compareTo(data2.getYear());
    }
}


//reduce
public static class MyReduce extends Reducer<Text,Week2Data,Text,Text>{
    @Override
    protected void reduce(Text key, Iterable<Week2Data> values, Context context) throws IOException, InterruptedException {
        for (Week2Data value : values){
            context.write(new Text(value.getYear()+"   "+value.getBalance()),new Text(""));
        }
    }
}

public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

    //job
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "week2-test2");
    //写入文件
    FileInputFormat.addInputPath(job,new Path(args[0]));
    //map计算
    job.setMapperClass(MyMapper.class);
    job.setMapOutputValueClass(Week2Data.class);
    job.setMapOutputKeyClass(Text.class);
    //shuffle流程
    //调用自定义分区器
    job.setPartitionerClass(MyPartitioner.class);
    //调用自定义分组
   // job.setGroupingComparatorClass(MyGroupComparble.class);
    //reduce计算
    //设置reduce分区个数
    job.setNumReduceTasks(2);
    job.setReducerClass(MyReduce.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    //判断文件是否存在
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(new Path(args[1]))) {
        fs.delete(new Path(args[1]), true);
    }
    //写出文件
    FileOutputFormat.setOutputPath(job,new Path(args[1]));
    //提交作业
    boolean result = job.waitForCompletion(true);
    System.out.println(result);
}

实体类
public class Week2Data implements WritableComparable{

private String year;
private Integer balance;

public Week2Data(String year, Integer balance) {
    this.year = year;
    this.balance = balance;
}

public Week2Data() {

}

public String getYear() {
    return year;
}

public void setYear(String year) {
    this.year = year;
}

public Integer getBalance() {
    return balance;
}

public void setBalance(Integer balance) {
    this.balance = balance;
}

@Override
public int compareTo(Week2Data o) {
    return o.balance.compareTo(this.getBalance());
}

@Override
public void write(DataOutput output) throws IOException {
    output.writeUTF(year);
    output.writeInt(balance);
}

@Override
public void readFields(DataInput input) throws IOException {
    this.year = input.readUTF();
    this.balance = input.readInt();
}

}