MapReduce 之-- 某个用户在某个位置从某个时刻开始停留了多长时间--升级版

package kaoshi831;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class MRsort {
    /**
     * 
     *
        数据意义：某个用户在某个位置从某个时刻开始停留了多长时间
        处理逻辑：
        对同一个用户，在同一个位置，连续的多条记录进行合并
        合并原则：开始时间取最早的，停留时长加和
        用户ID，位置ID，开始时间，停留时长（分钟）
     *  user_a,location_a,2018-01-01 08:00:00,60
     */
    static class MyMapper extends Mapper<LongWritable, Text, SortOwn, Text>{
        private Text mv=new Text();
        private SortOwn so = new SortOwn();
        @Override
        protected void map(LongWritable key,
                Text value, 
                Mapper<LongWritable, Text, SortOwn, Text>.Context context)
                throws IOException, InterruptedException {
            //user_a,location_a,2018-01-01 08:00:00,60
            String[] sp = value.toString().split(",");  //读取每行的元素   
            so.setUlid(sp[0]+"\t"+sp[1]);           //将字段，（用户ID，位置ID，开始时间）添加到包装类中
            so.setTime(sp[2]);                      //按照，（用户ID，位置ID）分组，（开始时间）排序
            mv.set(sp[3]);
            context.write(so, mv);
        }
    }
    static class MyReducer extends Reducer<SortOwn,Text , Text, IntWritable>{
        private Text outkey = new Text();
        private IntWritable outvalue= new IntWritable();
        @Override
        protected void reduce(SortOwn key,
                Iterable<Text> values, 
                Context context)
                throws IOException, InterruptedException {
            //user_a,location_a,2018-01-01 08:00:00,60
            int sum=0;  
            for(Text v:values){
                String sp = v.toString();
                sum+=Integer.parseInt(sp);
                System.out.println(key);        //测试key的位置--（包装类中我将"开始时间"按照倒序排列），所有取最后一个key.getTime();
            }                                   //values是个迭代器，类似指针的遍历方式
            String k=key.getUlid()+"\t"+key.getTime();
            outkey.set(k);
            outvalue.set(sum);
            context.write(outkey, outvalue);
        }
    }

    public static void main(String[] args) throws IllegalArgumentException, IOException, URISyntaxException, ClassNotFoundException, InterruptedException {
        System.setProperty("HADOOP_USER_NAME", "hadoop");
        Configuration conf=new Configuration();
        Job job=Job.getInstance(conf);

        job.setJarByClass(kaoshi831.MRsort.class);

        job.setMapperClass(MyMapper.class);
        job.setReducerClass(MyReducer.class);

        job.setMapOutputKeyClass(SortOwn.class);
        job.setMapOutputValueClass(Text.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        //添加分组
        job.setGroupingComparatorClass(MyGroup.class);

        FileInputFormat.addInputPath(job, new Path("hdfs://hadoop01:9000/ksin"));

        FileSystem fs = FileSystem.get(new URI("hdfs://hadoop01:9000"), conf);
        Path path=new Path("hdfs://hadoop01:9000/ksout01");
        if(fs.exists(path)){
            fs.delete(path, true);
        }
        FileOutputFormat.setOutputPath(job,path);

        job.waitForCompletion(true);

    }

}

//自定义包装类，实现WritableComparable

package kaoshi831;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.WritableComparable;

public class SortOwn implements WritableComparable<SortOwn>{
    //user_a,location_a,2018-01-01 08:00:00,60
    private String ulid;
    private String time;

    @Override
    public String toString() {
        return ulid + "," + time;
    }
    public SortOwn() {
        super();
        // TODO Auto-generated constructor stub
    }
    public String getUlid() {
        return ulid;
    }
    public void setUlid(String ulid) {
        this.ulid = ulid;
    }
    public String getTime() {
        return time;
    }
    public void setTime(String time) {
        this.time = time;
    }
    public SortOwn(String ulid, String time) {
        super();
        this.ulid = ulid;
        this.time = time;
    }
    @Override
    public void write(DataOutput out) throws IOException {
        out.writeUTF(ulid);
        out.writeUTF(time);

    }
    @Override
    public void readFields(DataInput in) throws IOException {
        this.ulid=in.readUTF();
        this.time=in.readUTF();
    }
    @Override
    public int compareTo(SortOwn o) {
        int tmp=this.ulid.compareTo(o.ulid);
        if(tmp==0){
            return o.time.compareTo(this.time);//倒序
        }
        return tmp;
    }

}

//自定义分组 继承WritableComparator

package kaoshi831;

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

public class MyGroup extends WritableComparator {
    public MyGroup(){//通过构造加载
        super(SortOwn.class,true);//用反射，加载要分组的类
    }
    @Override
    public int compare(WritableComparable a, WritableComparable b) {
        //定义分组条件
        SortOwn sb1=(SortOwn)a;
        SortOwn sb2=(SortOwn)b;
            return sb1.getUlid().compareTo(sb2.getUlid());
    }
}