原理是每个map保留TopN数据,然后发送给reduce, reduce只设置一个,也就是说如果有10个map, 那么每个map处理好数据保留TopN列表,然后10个map就是100条数据,然后发送给reduce. 然后由reduce来过滤出10条数据。
每个Mapper开始之前有setup方法,结束之后有cleanup, 通常不会使用,setup, cleanup仅仅是每个mapper开始或者结束后才调用,仅仅调用一次。 上述算法原理是map来过滤出数据,然后统一交个cleanup发送给reduce.
reduce层原理也一样,假设10个map数据传给reduce,那么会有reduce会接受top100条数据,然后交给cleanup一次性过滤,然后output. 这也是为什么仅仅设置一个reduce的原因。
原始数据如下:
30;"unemployed";"married";"primary";"no";1787;"no";"no";"cellular";19;"oct";79;1;-1;0;"unknown";"no" 33;"services";"married";"secondary";"no";4789;"yes";"yes";"cellular";11;"may";220;1;339;4;"failure";"no" 35;"management";"single";"tertiary";"no";1350;"yes";"no";"cellular";16;"apr";185;1;330;1;"failure";"no" 30;"management";"married";"tertiary";"no";1476;"yes";"yes";"unknown";3;"jun";199;4;-1;0;"unknown";"no" 59;"blue-collar";"married";"secondary";"no";0;"yes";"no";"unknown";5;"may";226;1;-1;0;"unknown";"no" 35;"management";"single";"tertiary";"no";747;"no";"no";"cellular";23;"feb";141;2;176;3;"failure";"no" 36;"self-employed";"married";"tertiary";"no";307;"yes";"no";"cellular";14;"may";341;1;330;2;"other";"no" 39;"technician";"married";"secondary";"no";147;"yes";"no";"cellular";6;"may";151;2;-1;0;"unknown";"no" 41;"entrepreneur";"married";"tertiary";"no";221;"yes";"no";"unknown";14;"may";57;2;-1;0;"unknown";"no" 43;"services";"married";"primary";"no";-88;"yes";"yes";"cellular";17;"apr";313;1;147;2;"failure";"no" 39;"services";"married";"secondary";"no";9374;"yes";"no";"unknown";20;"may";273;1;-1;0;"unknown";"no" 43;"admin.";"married";"secondary";"no";264;"yes";"no";"cellular";17;"apr";113;2;-1;0;"unknown";"no" 36;"technician";"married";"tertiary";"no";1109;"no";"no";"cellular";13;"aug";328;2;-1;0;"unknown";"no" 20;"student";"single";"secondary";"no";502;"no";"no";"cellular";30;"apr";261;1;-1;0;"unknown";"yes" 31;"blue-collar";"married";"secondary";"no";360;"yes";"yes";"cellular";29;"jan";89;1;241;1;"failure";"no" 40;"management";"married";"tertiary";"no";194;"no";"yes";"cellular";29;"aug";189;2;-1;0;"unknown";"no" 56;"technician";"married";"secondary";"no";4073;"no";"no";"cellular";27;"aug";239;5;-1;0;"unknown";"no" 37;"admin.";"single";"tertiary";"no";2317;"yes";"no";"cellular";20;"apr";114;1;152;2;"failure";"no" 25;"blue-collar";"single";"primary";"no";-221;"yes";"no";"unknown";23;"may";250;1;-1;0;"unknown";"no" 31;"services";"married";"secondary";"no";132;"no";"no";"cellular";7;"jul";148;1;152;1;"other";"no" 38;"management";"divorced";"unknown";"no";0;"yes";"no";"cellular";18;"nov";96;2;-1;0;"unknown";"no" 42;"management";"divorced";"tertiary";"no";16;"no";"no";"cellular";19;"nov";140;3;-1;0;"unknown";"no" 44;"services";"single";"secondary";"no";106;"no";"no";"unknown";12;"jun";109;2;-1;0;"unknown";"no".............................. 上面只是数据样例,实际还有很多行.
实现代码如下:
package com.isesol.mapreduce;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.SortedMap;
import java.util.StringTokenizer;
import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.commons.net.nntp.NewsgroupInfo;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hadoop.mapred.TextOutputFormat;
import org.apache.hadoop.mapred.lib.NullOutputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.OutputCommitter;
import org.apache.hadoop.mapreduce.OutputFormat;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.Reducer.Context;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SplitLineReader;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs;
public class topN {
public static class TokenizerMapper extends Mapper<Object, Text, Text, Text> {
private SortedMap top10 = new TreeMap();
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
String str = "";
String[] val = value.toString().split(";");
for (int i = 1; i < val.length - 1; i++) {
str += val[i];
}
top10.put(val[0], str);
if (top10.size() > 10) {
top10.remove(top10.firstKey());
}
}
public void cleanup(Context context) throws IOException, InterruptedException {
Iterator iterator = top10.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry ent = (Map.Entry) iterator.next();
String key = ent.getKey().toString();
String value = ent.getValue().toString();
context.write(new Text(key), new Text(value));
}
}
}
public static class IntSumReducer extends Reducer<Text, Text, Text, Text> {
private SortedMap top10 = new TreeMap();
public void reduce(Text key, Iterable<Text> value, Context context) throws IOException, InterruptedException {
for( Text val : value) {
top10.put(key.toString(), val.toString());
}
}
public void cleanup(Context context) throws IOException, InterruptedException {
System.out.println("top10 size isa: " + top10.size());
while(top10.size() > 10) {
top10.remove(top10.firstKey());
}
Iterator iterator = top10.entrySet().iterator();
System.out.println("top10 size is: " + top10.size());
while (iterator.hasNext()) {
Map.Entry ent = (Map.Entry) iterator.next();
String key01 = ent.getKey().toString();
String value01 = ent.getValue().toString();
context.write(new Text(key01), new Text(value01));
}
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "topN");
job.setJarByClass(topN.class);
job.setMapperClass(TokenizerMapper.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setNumReduceTasks(1);
// job.setPartitionerClass(twopartitions.class);
// job.setOutputFormatClass(fakeOutPutFormat.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
结果如下:
76 "retired""married""primary""no"2590"no""no""telephone"9"feb"6812-10"unknown" 77 "retired""married""primary""no"680"no""no""telephone"27"nov"3414943"failure" 78 "retired""married""tertiary""no"226"no""no""telephone"6"nov"1361-10"unknown" 79 "retired""divorced""unknown""no"2628"no""no""telephone"8"jul"22074502"failure" 80 "housemaid""married""primary""no"0"no""no""cellular"23"feb"63911891"failure" 81 "retired""married""secondary""no"1"no""no""cellular"19"aug"655-10"unknown" 83 "retired""divorced""primary""no"1097"no""no""telephone"5"mar"1811-10"unknown" 84 "retired""divorced""primary""no"639"no""no""telephone"18"may"3533-10"unknown" 86 "retired""married""secondary""no"1503"no""no""telephone"18"mar"16531011"other" 87 "retired""married""primary""no"230"no""no""cellular"30"oct"1441-10"unknown"上面没有考虑有重复数据,如果需要考虑重复数据也很好处理,稍微加工一下即可。
上面的TOP N是固定的,通常在实际些程序不会把N固定,比如我想TOP 100,不可能再去修改参数,因此赋予变量是最好的方式。Mpreduce通过conf.set(name, value)的方式设置变量,然后通过get获取变量。
conf.set("topn", args[2]);
topn = context.getConfiguration().get("topn");
if(topn == null) {
topn = "10";
}