{“movie”:“1193”,“rate”:“5”,“timeStamp”:“978300760”,“uid”:“1”}
{“movie”:“661”,“rate”:“3”,“timeStamp”:“978302109”,“uid”:“1”}
{“movie”:“914”,“rate”:“3”,“timeStamp”:“978301968”,“uid”:“1”}
{“movie”:“3408”,“rate”:“4”,“timeStamp”:“978300275”,“uid”:“1”}
… …
求最热门的前三个电影,即评论次数最多的
定义一个MovieBean,实现Hadoop的序列化接口
import org.apache.hadoop.io.Writable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class MovieBean implements Writable {
private String movie;
private String rate;
private String timeStamp;
private String uid;
public MovieBean() {
}
public MovieBean(String movie, String rate, String timeStamp, String uid) {
this.movie = movie;
this.rate = rate;
this.timeStamp = timeStamp;
this.uid = uid;
}
public String getMovie() {
return movie;
}
public void setMovie(String movie) {
this.movie = movie;
}
public String getRate() {
return rate;
}
public void setRate(String rate) {
this.rate = rate;
}
public String getTimeStamp() {
return timeStamp;
}
public void setTimeStamp(String timeStamp) {
this.timeStamp = timeStamp;
}
public String getUid() {
return uid;
}
public void setUid(String uid) {
this.uid = uid;
}
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeUTF(this.movie);
dataOutput.writeUTF(this.rate);
dataOutput.writeUTF(this.timeStamp);
dataOutput.writeUTF(this.uid);
}
public void readFields(DataInput dataInput) throws IOException {
this.movie = dataInput.readUTF();
this.rate = dataInput.readUTF();
this.timeStamp = dataInput.readUTF();
this.uid = dataInput.readUTF();
}
@Override
public String toString() {
return "MovieBean{" +
"movie='" + movie + '\'' +
", rate='" + rate + '\'' +
", timeStamp='" + timeStamp + '\'' +
", uid='" + uid + '\'' +
'}';
}
}
mapredece
//需求:最热的三部电影(即评论次数最多的)
public class MovieTopn {
static class MovieMapper extends Mapper<LongWritable, Text, Text, MovieBean> {
Text movie = new Text();
Gson gs = new Gson();
@Override
protected void map(LongWritable key, Text value, Context context) {
try {
//解析读到的一行电影评分记录
// MovieBean mb = JSON.parseObject(value.toString(), MovieBean.class);
MovieBean mb = gs.fromJson(value.toString(), MovieBean.class);
//获取电影id
movie.set(mb.getMovie());
//电影id作为键,评分记录作为value
context.write(movie, mb);
} catch (Exception e) {
e.printStackTrace();
}
}
}
static class MovieReducer extends Reducer<Text, MovieBean, Text, IntWritable> {
Map<String,Integer> map = new HashMap<String, Integer>();
@Override
protected void reduce(Text key, Iterable<MovieBean> values, Context context) throws IOException, InterruptedException {
int num = 0;
for (MovieBean value : values) {
num++;
}
//添加每部电影的评论次数
map.put(key.toString(), num);
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
//map按照评论次数排序
Set<Map.Entry<String, Integer>> entrySet = map.entrySet();
ArrayList<Map.Entry<String, Integer>> list = new ArrayList<Map.Entry<String, Integer>>(entrySet);
list.sort(new Comparator<Map.Entry<String, Integer>>() {
public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
return o2.getValue() - o1.getValue();
}
});
for (int i = 0; i < Math.min(3, list.size()); i++) {
Map.Entry<String, Integer> res = list.get(i);
String mid = res.getKey();
Integer cnt = res.getValue();
context.write(new Text(mid),new IntWritable(cnt));
}
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "movieTop");
//设置map reduce 逻辑类
job.setMapperClass(MovieMapper.class);
job.setReducerClass(MovieReducer.class);
//设置输出类
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(MovieBean.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
//设置数据输入 输出
FileInputFormat.setInputPaths(job, new Path("D:\\MR\\movie\\input"));
FileOutputFormat.setOutputPath(job, new Path("D:\\MR\\movie\\output2"));
job.waitForCompletion(true);
}
}