mapreduce常见案例

常见案例

1.二次排序
2.共同好友
3.点击流分析模型

案例一：二次排序

原始数据

目标数据

代码
–Map/Reduce不需要改变，主要是SortComparator通过继承WritableComparator类，重写了compare方法，改变了hadoop默认的排序规则。

步骤1.建立对象存储数据，继承WritableComparable，添加序列化、反序列化、比较器

package com.buaa;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

import org.apache.hadoop.io.WritableComparable;

public class IntPair implements WritableComparable<IntPair>{
    private int first;
    private int second;
    
    public IntPair(){
    }
    
    public IntPair(int left, int right){
        set(left, right);
    }
    
    public void set(int left, int right){
        first = left;
        second = right;
    }
    
    @Override
    public void readFields(DataInput in) throws IOException{
        first = in.readInt();
        second = in.readInt();
    }
    
    @Override
    public void write(DataOutput out) throws IOException{
        out.writeInt(first);
        out.writeInt(second);
    }
    
    @Override
    public int compareTo(IntPair o)
    {
        if (first != o.first){
            return first < o.first ? -1 : 1;
        }else if (second != o.second){
            return second < o.second ? -1 : 1;
        }else{
            return 0;
        }
    }
    
    @Override
    public int hashCode(){
        return first * 157 + second;
    }
    
    @Override
    public boolean equals(Object right){
        if (right == null)
            return false;
        if (this == right)
            return true;
        if (right instanceof IntPair){
            IntPair r = (IntPair) right;
            return r.first == first && r.second == second;
        }else{
            return false;
        }
    }
    
    public int getFirst(){
        return first;
    }
    
    public int getSecond(){
        return second;
    }
}

步骤2.实现分区内排序

复制代码
package com.buaa;

import java.io.IOException;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apac