MapperReducer简单使用案例分享

显示记录时按照每个用户的评分从高到低显示 {"movie":"1193","rate":5.0,"timeStamp":978300760,"uid":"1"} {"movie":"661","rate":4.0,"timeStamp":978302109,"uid":"1"} {"movie":"661","rate":1.0,"timeStamp":978302109,"uid":"1"} {"movie":"661","rate":1.0,"timeStamp":978302109,"uid":"1"} {"movie":"661","rate":1.0,"timeStamp":978302109,"uid":"1"} {"movie":"661","rate":3.0,"timeStamp":978302109,"uid":"2"} {"movie":"1193","rate":3.0,"timeStamp":978300760,"uid":"2"} {"movie":"1193","rate":2.0,"timeStamp":978300760,"uid":"2"} {"movie":"1193","rate":2.0,"timeStamp":978300760,"uid":"2"} {"movie":"1193","rate":5.0,"timeStamp":978300760,"uid":"3"} {"movie":"1193","rate":5.0,"timeStamp":978300760,"uid":"3"} {"movie":"661","rate":4.0,"timeStamp":978302109,"uid":"3"} {"movie":"1193","rate":3.0,"timeStamp":978300760,"uid":"3"} {"movie":"661","rate":4.0,"timeStamp":978302109,"uid":"5"}

package day35.Test;

import com.google.gson.Gson;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;

public class Test {
    private static class MovieMapper extends Mapper<LongWritable, Text,Text,MovieBean>{
        Gson gs = new Gson();
        Text k2 = new Text();

        @Override
        protected void map  (LongWritable key, Text value, Mapper<LongWritable, Text, Text, MovieBean>.Context context) throws IOException, InterruptedException {
            MovieBean mb = gs.fromJson(value.toString(), MovieBean.class);
            String movie = mb.getMovie();
            k2.set(movie);
            context.write(k2,mb);
        }
    }
    private static class MovieReducer extends Reducer<Text,MovieBean, NullWritable,Text>{
        Text v3 = new Text();

        @Override
        protected void reduce(Text key, Iterable<MovieBean> values, Reducer<Text, MovieBean, NullWritable, Text>.Context context) throws IOException, InterruptedException {
            ArrayList<MovieBean> Marr = new ArrayList<>();
            for (MovieBean value : values) {
                MovieBean mb = new MovieBean();
                mb.setUid(value.getUid());
                mb.setMovie(value.getMovie());
                mb.setRate(value.getRate());
                mb.setTimeStamp(value.getTimeStamp());
                Marr.add(mb);
            }
            Collections.sort(Marr, new Comparator<MovieBean>() {
                @Override
                public int compare(MovieBean o1, MovieBean o2) {
                    return Double.compare(o1.getRate(), o2.getRate());
                }
            });
            for (MovieBean movieBean : Marr) {
                Gson gs =new Gson();
                v3.set(gs.toJson(movieBean));
                context.write(NullWritable.get(),v3);
            }

        }
    }

    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        Configuration con = new Configuration();

        Job job = Job.getInstance(con,"movie3");

        job.setMapperClass(MovieMapper.class);
        job.setReducerClass(MovieReducer.class);

        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(MovieBean.class);

        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(Text.class);

        FileInputFormat.setInputPaths(job,new Path("要读取的文件路径"));
        FileOutputFormat.setOutputPath(job,new Path("输出的文件路径"));

        job.waitForCompletion(true);

    }
}

以上需要注意的是Reducer的输出并不需要一个特定的键值，而是只返回每个用户对电影的评级，并按照评分排序后输出。由于这个输出的键值对没有实际意义，因此使用了NullWritable作为输出键，而将评级的信息作为输出值，以便更好地表示这种情况。