MapReduce之气温计算

Step 1：
导入Hadoop和Mapreduce的所有jar包

Step 2：WeatherMapper

public class WeatherMapper extends Mapper<Text, Text, MyKey, Text>{

    @Override
    protected void map(Text key, Text value,
            Context context)
            throws IOException, InterruptedException {

        try {
            // 1950-10-01 12:21:02  37c
            SimpleDateFormat sdf =new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            Date date =sdf.parse(key.toString());
            Calendar c =Calendar.getInstance();
            c.setTime(date);
            int year =c.get(Calendar.YEAR);
            int month =c.get(Calendar.MONTH);
            int temp =Integer.parseInt(value.toString().substring(0, value.toString().lastIndexOf("c")));

            MyKey mykey = new MyKey();
            mykey.setYear(year);
            mykey.setMonth(month);
            mykey.setTemp(temp);

            context.write(mykey, value);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

Step 3：Shffle阶段自定义Group

public class MyGroup extends WritableComparator{

    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        MyKey k1 =(MyKey) a;
        MyKey k2 =(MyKey) b;
        int r1 =Integer.compare(k1.getYear(), k2.getYear());
        if(r1==0){
            return Integer.compare(k1.getMonth(), k2.getMonth());
        }else{
            return r1;
        }
    }

    public MyGroup() {
        super(MyKey.class,true);
    }
}

Step 4：shuffle阶段自定义key

public class MyKey implements WritableComparable<MyKey>{

    private int year;
    private int month;
    private int temp;
    public int getYear() {
        return year;
    }
    public void setYear(int year) {
        this.year = year;
    }
    public int getMonth() {
        return month;
    }
    public void setMonth(int month) {
        this.month = month;
    }
    public int getTemp() {
        return temp;
    }
    public void setTemp(int temp) {
        this.temp = temp;
    }
    @Override
    public void readFields(DataInput in) throws IOException {
        this.year = in.readInt();
        this.month = in.readInt();
        this.temp = in.readInt();
    }
    @Override
    public void write(DataOutput out) throws IOException {
        out.writeInt(year);
        out.writeInt(month);
        out.writeInt(temp);
    }
    @Override
    public int compareTo(MyKey o) {
        int r1 =Integer.compare(this.year, o.getYear());
        if(r1==0){
            int r2 =Integer.compare(this.month, o.getMonth());
            if(r2==0){
                return  Double.compare(this.temp, o.getTemp());
            }else{
                return r2;
            }
        }else{
            return r1;
        }
    }

}

Step 5：shuffle阶段自定义partition

public class MyPartitioner extends HashPartitioner<MyKey, Text>{

    @Override
    public int getPartition(MyKey key, Text value, int numReduceTasks) {
        return (key.getYear()-1949) % numReduceTasks;
    }

}

Step 6：shuffle阶段自定义排序

public class MySort extends WritableComparator{

    public MySort() {
        super(MyKey.class, true);
    }

    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        MyKey k1 =(MyKey) a;
        MyKey k2 =(MyKey) b;
        int r1 =Integer.compare(k1.getYear(), k2.getYear());
        if(r1==0){
            int r2 =Integer.compare(k1.getMonth(), k2.getMonth());
            if(r2==0){
                return -Double.compare(k1.getTemp(), k2.getTemp());
            }else{
                return r2;
            }
        }else{
            return r1;
        }
    }


}

Step 7：shuffle阶段自定义combiner

public class WeatherCombiner extends Reducer<MyKey, Text, MyKey, Text>{

    @Override
    protected void reduce(MyKey key, Iterable<Text> values,
            Context context) throws IOException,
            InterruptedException {

        int i = 0;
        for(Text value : values){
            context.write(key, value);
            if(i==3){
                break;
            }
            i++;
        }
    }


}

Step 8：weather之reducer阶段

public class WeatherReducer extends Reducer<MyKey, Text, Text, NullWritable>{

    @Override
    protected void reduce(MyKey key, Iterable<Text> values,
            Context context) throws IOException,
            InterruptedException {

        int i = 0;
        for(Text value : values){
            context.write(new Text(key.getYear()+"\t"+key.getMonth()+"\t"+value), NullWritable.get());
            if(i==3){
                break;
            }
            i++;
        }
    }


}

Step 9：MapReduce之Main—-RunJob

public class RunJob {

    public static void main(String[] args) {
        try {
            int numReduceTasks = Integer.valueOf(args[0]);

            Configuration conf = new Configuration();
            FileSystem fs = FileSystem.newInstance(conf);

            Job job = Job.getInstance();
            job.setJarByClass(RunJob.class);

            job.setJobName("weather");
            job.setInputFormatClass(KeyValueTextInputFormat.class);

            job.setMapOutputKeyClass(MyKey.class);
            job.setMapOutputValueClass(Text.class);     

            job.setMapperClass(WeatherMapper.class);
            job.setReducerClass(WeatherReducer.class);
//          job.setCombinerClass(WeatherCombiner.class);

            job.setNumReduceTasks(numReduceTasks);

            job.setGroupingComparatorClass(MyGroup.class);
            job.setPartitionerClass(MyPartitioner.class);
            job.setSortComparatorClass(MySort.class);

//          job.setCombinerClass(cls);

            FileInputFormat.addInputPath(job, new Path("/data/weather"));
            Path output = new Path("/weather");
            if(fs.exists(output)){
                fs.delete(output, true);
            }
            FileOutputFormat.setOutputPath(job, output);

            boolean flag = job.waitForCompletion(true);
            if(flag){
                System.out.println("Job finished !");
            }

        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}