–Reducer
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class HotReduce extends Reducer<KeyPair, Text, KeyPair, Text>{
@Override
protected void reduce(KeyPair kp, Iterable<Text> i,Context context)
throws IOException, InterruptedException {
for (Text text : i) {
context.write(kp, text);
}
}
}
–Mapper
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class HotMapper extends Mapper<LongWritable, Text, KeyPair, Text>{
@Override
protected void map(LongWritable key, Text value,Context context)
throws IOException, InterruptedException {
String line=value.toString();
String[] ss=line.split("\t");
if (ss.length==2){
int year=Integer.parseInt(ss[0].substring(0, 4));
int hot=Integer.parseInt(ss[1].substring(0, ss[1].indexOf("°C")));
KeyPair kp=new KeyPair();
kp.setYear(year);
kp.setHot(hot);
context.write(kp, value);
}
}
}
–KeyPair 自定义封装类
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.WritableComparable;
public class KeyPair implements WritableComparable<KeyPair> {
private int year;
private int hot;
public int getYear() {
return year;
}
public void setYear(int year) {
this.year = year;
}
public int getHot() {
return hot;
}
public void setHot(int hot) {
this.hot = hot;
}
@Override 反序列化
public void readFields(DataInput in) throws IOException {
this.year=in.readInt();
this.hot=in.readInt();
}
@Override 序列化
public void write(DataOutput out) throws IOException {
out.writeInt(year);
out.writeInt(hot);
}
@Override 对比
public int compareTo(KeyPair o) {
int result=Integer.compare(year, o.getYear());
if (result!=0){
return result;
}
return Integer.compare(hot, o.hot);
}
@Override
public String toString() {
return year+"\t"+hot;
}
@Override
public int hashCode() {
return new Integer(year+hot).hashCode();
}
}
–HotPartition
//自定义分区
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;
public class HotPartition extends Partitioner<KeyPair, Text>{
@Override
public int getPartition(KeyPair key, Text value, int num) {
return (key.getYear()*127%num);
}
}
–HotSort 自定义排序
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
public class HotSort extends WritableComparator{
public HotSort() {
super(KeyPair.class, true); //排序时使用自定义分装的类进行排序
}
@Override
public int compare(WritableComparable a, WritableComparable b) {
KeyPair o1=(KeyPair) a;
KeyPair o2=(KeyPair) b;
int res=Integer.compare(o1.getYear(), o2.getYear());
if (res!=0){
return res;
}
return -Integer.compare(o1.getHot(),o2.getHot());//降序排序
}
}
–HotGroup
自定义分组
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
public class HotGroup extends WritableComparator{
public HotGroup() {
super(KeyPair.class, true);
}
@Override
public int compare(WritableComparable a, WritableComparable b) {
KeyPair o1=(KeyPair) a;
KeyPair o2=(KeyPair) b;
return Integer.compare(o1.getYear(),o2.getYear());
}
}
public class Job{
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration configuration = new Configuration();
Job job = new Job(configuration,"partation");
job.setJarByClass(Job.class);
job.setMapperClass(HotMapper.class);
job.setReducerClass(HotReducer.class);
job.setOutputKeyClass(KeyPair.class);
job.setOutputValueClass(Text.class);
job.setGroupingComparatorClass(FirstPartation.class);
job.setNumReduceTasks(3);
job.setSortComparatorClass(SortHot.class);
job.setGroupingComparatorClass(GroupHot.class);
FileInputFormat.addInputPath(job, new Path("/user/root/books"));
FileOutputFormat.setOutputPath(job, new Path("/user/root/bookout"));
System.exit(job.waitForCompletion(true)?0:1);
}
}