电影TopN案例——mapreduce

{“movie”:“1193”,“rate”:“5”,“timeStamp”:“978300760”,“uid”:“1”}
{“movie”:“661”,“rate”:“3”,“timeStamp”:“978302109”,“uid”:“1”}
{“movie”:“914”,“rate”:“3”,“timeStamp”:“978301968”,“uid”:“1”}
{“movie”:“3408”,“rate”:“4”,“timeStamp”:“978300275”,“uid”:“1”}
… …
求最热门的前三个电影，即评论次数最多的

定义一个MovieBean，实现Hadoop的序列化接口

import org.apache.hadoop.io.Writable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class MovieBean implements Writable {
    private String movie;
    private String rate;
    private String timeStamp;
    private String uid;


    
    public MovieBean() {
    }

    public MovieBean(String movie, String rate, String timeStamp, String uid) {
        this.movie = movie;
        this.rate = rate;
        this.timeStamp = timeStamp;
        this.uid = uid;
    }

    public String getMovie() {
        return movie;
    }

    public void setMovie(String movie) {
        this.movie = movie;
    }

    public String getRate() {
        return rate;
    }

    public void setRate(String rate) {
        this.rate = rate;
    }

    public String getTimeStamp() {
        return timeStamp;
    }

    public void setTimeStamp(String timeStamp) {
        this.timeStamp = timeStamp;
    }

    public String getUid() {
        return uid;
    }

    public void setUid(String uid) {
        this.uid = uid;
    }



    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeUTF(this.movie);
        dataOutput.writeUTF(this.rate);
        dataOutput.writeUTF(this.timeStamp);
        dataOutput.writeUTF(this.uid);
    }

    public void readFields(DataInput dataInput) throws IOException {
        this.movie = dataInput.readUTF();
        this.rate = dataInput.readUTF();
        this.timeStamp = dataInput.readUTF();
        this.uid = dataInput.readUTF();

    }

    @Override
    public String toString() {
        return "MovieBean{" +
                "movie='" + movie + '\'' +
                ", rate='" + rate + '\'' +
                ", timeStamp='" + timeStamp + '\'' +
                ", uid='" + uid + '\'' +
                '}';
    }
}

mapredece

//需求：最热的三部电影（即评论次数最多的）
public class MovieTopn {
    static class MovieMapper extends Mapper<LongWritable, Text, Text, MovieBean> {
        Text movie = new Text();
        Gson gs = new Gson();

        @Override
        protected void map(LongWritable key, Text value, Context context) {

            try {
                //解析读到的一行电影评分记录
//                MovieBean mb = JSON.parseObject(value.toString(), MovieBean.class);
                MovieBean mb = gs.fromJson(value.toString(), MovieBean.class);
                //获取电影id
                movie.set(mb.getMovie());
                //电影id作为键，评分记录作为value
                context.write(movie, mb);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    static class MovieReducer extends Reducer<Text, MovieBean, Text, IntWritable> {
        Map<String,Integer> map = new HashMap<String, Integer>();


        @Override
        protected void reduce(Text key, Iterable<MovieBean> values, Context context) throws IOException, InterruptedException {
            int num = 0;
            for (MovieBean value : values) {
                num++;

            }
            //添加每部电影的评论次数
            map.put(key.toString(), num);
        }

        @Override
        protected void cleanup(Context context) throws IOException, InterruptedException {
            //map按照评论次数排序
            Set<Map.Entry<String, Integer>> entrySet = map.entrySet();
            ArrayList<Map.Entry<String, Integer>> list = new ArrayList<Map.Entry<String, Integer>>(entrySet);
            list.sort(new Comparator<Map.Entry<String, Integer>>() {
                public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
                    return o2.getValue() - o1.getValue();
                }
            });

            for (int i = 0; i < Math.min(3, list.size()); i++) {
                Map.Entry<String, Integer> res = list.get(i);
                String mid = res.getKey();
                Integer cnt = res.getValue();
                context.write(new Text(mid),new IntWritable(cnt));

            }
        }


    }

    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf, "movieTop");

        //设置map reduce 逻辑类
        job.setMapperClass(MovieMapper.class);
        job.setReducerClass(MovieReducer.class);

        //设置输出类
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(MovieBean.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);


        //设置数据输入 输出
        FileInputFormat.setInputPaths(job, new Path("D:\\MR\\movie\\input"));
        FileOutputFormat.setOutputPath(job, new Path("D:\\MR\\movie\\output2"));


        job.waitForCompletion(true);


    }

}