MapReduce编程实例之source-destination-count-avertime

//Hw2Part1.java

/**
 * Statistic the average time and the number of communication between source and destination 
 * Author:Rosun
 * date: 2017/4/8
 */
import java.io.IOException;
import java.math.*;
import java.util.StringTokenizer;
import java.io.DataInput;
import java.io.DataOutput;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
/**
 * set value of count and communication's time
 *
 */
class MymulvalueWritable implements Writable{
    //value 
    public int count;
    public float avertime;

    public void set(int count,float avertime){
        this.count=count;
        this.avertime=avertime;
    }
    public int getCount(){
        return this.count;
    }
    public float getAvertime(){
        return this.avertime;
    }
    @Override
    public String toString(){
        return count+"\t"+avertime+"\t";
    }
    @Override
    public void readFields(DataInput in )throws IOException{
        this.count=in.readInt();
        this.avertime=in.readFloat();

    }
    @Override
    public void write(DataOutput out )throws IOException{
        out.writeInt(count);
        out.writeFloat(avertime);

    }

}

public class Hw2Part1{
    public Hw2Part1(){
        System.out.println("excute constructed function--Hw2Part1()");

    }

  /**
   * this class extends Mapper,split input date ,(ik,iv)->(mk,mv)
   * @author guest
   */
  public static class TokenizerMapper 
                extends Mapper<Object, Text, Text, MymulvalueWritable>{
    public MymulvalueWritable result = new MymulvalueWritable();
    private Text source_destination = new Text();
    public void map(Object key, Text value, Context context
                    ) throws IOException, InterruptedException {
          System.out.println("Excute map() in TokenizerMapper Class");
          //get every line
          StringTokenizer itr = new StringTokenizer(value.toString(),"\n");
          while (itr.hasMoreTokens()) {
              String[] itr1=itr.nextToken().split("\\s+");//get strings[] from itr
              //record format legal
              try{
                  if(itr1.length==3 && Float.parseFloat(itr1[2])>=0.0 &&itr1[2].matches("\\d+(.\\d+)?[fF]?")){
                      System.out.println(itr1[0]+"     "+itr1[1]);
                      source_destination.set(itr1[0]+"     "+itr1[1]);//m-key  
                      //System.out.println("time="+Float.parseFloat(itr1[2]));//holding time 
                      result.set(1,Float.parseFloat(itr1[2]));//m-value  count time
                      context.write(source_destination,result);
                      System.out.println(result.toString());
                  }
              }catch(Exception e){

              }
              else{
                   // throw away the record 
              }
          }
    }
  }

  /**
   * this class extens Reducer Class,statistic (mk,list(mv) ) on single block
   * @author guest
   *
   */
  public static class IntSumCombiner
                extends Reducer<Text,MymulvalueWritable,Text,MymulvalueWritable> {
        //  System.out.println("into IntSumCombiner");
        public  MymulvalueWritable result =new MymulvalueWritable();
        public void reduce(Text key, Iterable<MymulvalueWritable> values,Context context) 
                    throws IOException, InterruptedException {
              float aver = 0.0f;
        System.out.println("into IntSumCombiner");
              int i=0;
              System.out.println("Excute reduce() in IntSumCombiner Class");
              System.out.println("Excute sum by key");
              for (MymulvalueWritable val : values) {
                  aver += val.getAvertime();
                  i+=val.getCount();
              }
              //if(i!=0)
                //  aver=aver/i;
              System.out.println("key="+key.toString());
              //count and avertime
              result.set(i,aver);
              context.write(key, result);
              System.out.println(result.toString());
        }
  }

  /**
   * this class extends Reducer: (mk,mv)->(ok,ov)
   * @author guest
   */
  public static class IntSumReducer
                extends Reducer<Text,MymulvalueWritable,Text,Text> {

        private Text result_key= new Text();
        private Text result_value= new Text();
        public void reduce(Text key, Iterable<MymulvalueWritable> values, Context context)
                    throws IOException, InterruptedException {

              float aver = 0.0f;
              int i=0;
              System.out.println("Excute reduce() in IntSumReducer Class");
              //sum by key
              for (MymulvalueWritable val : values) {
                //aver+= val.getAvertime()*val.getCount();
                  aver+= val.getAvertime();
                i+=val.getCount();
              }
              //get the average holding time
              if(i!=0)
                  aver=aver/i;
              //keep 3 decimal places
              BigDecimal b=new BigDecimal(aver);
              float aver1=b.setScale(3,BigDecimal.ROUND_HALF_UP).floatValue();//convert 0.00 float 

              // generate result key
              result_value.set(String.valueOf(i)+"\t"+String.valueOf(aver1));
              context.write(key, result_value);

              System.out.println("result_key=key   "+key.toString());
              System.out.println("result_value="+result_value.toString());

        }
  }

  public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();
        System.out.println("conf information:  "+conf.toString());
        String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
        System.out.println("otherArgs:");
        for(int i=0;i<otherArgs.length;i++)
            System.out.println(otherArgs[i]);
        if (otherArgs.length < 2) {
          System.err.println("Usage: wordcount <in> [<in>...] <out>");
          System.exit(2);
        }
        Job job = Job.getInstance(conf, "average holding time");
        job.setJarByClass(Hw2Part1.class);
        job.setMapperClass(TokenizerMapper.class);
        job.setCombinerClass(IntSumCombiner.class);
        job.setReducerClass(IntSumReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(MymulvalueWritable.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        // add the input paths as given by command line
        for (int i = 0; i < otherArgs.length - 1; ++i) {
          FileInputFormat.addInputPath(job, new Path(otherArgs[i]));
          System.out.println(otherArgs[i]);
        }
        System.out.println(otherArgs[otherArgs.length - 1]);
        // add the output path as given by the command line
        FileOutputFormat.setOutputPath(job,new Path(otherArgs[otherArgs.length - 1]));
        System.exit(job.waitForCompletion(true) ? 0 : 1);
   }
}