1. 前言
需求:对输入文件中的数据进行排序,输入文件中的每一行均为一个数字,即为一个数据。要求在输出文件中每行输出两个数字,第一个代表原始数据在数据集中的顺次,第二个代表原始数据。
2. MapReduce实现排序的原理
在MapReduce中默认可以进行排序。
- 如果key是封装为IntWritable类型,那么MapReduce按照数字大小对key进行排序。
- 如果key是封装为String的Text类型,那么MapReduce按照字典顺序对字符串排序。
默认排序规则:
按照key值进行排序,所以应该使用封装int的IntWritable型数据结构,也就是在map中将读入的数据转化成IntWritable型,然后作为key值输出(value是任意的),reduce拿到<key, value-list>之后,将输入的key作为value输出,并根据value-list中元素的个数决定输出的次数。输出的key是一个全局变量,它统计当前key的顺次。
3. 上传文件
一通乱敲:
hadoop fs -put sort /sort
4. 代码实现
package com.mapreduce.sort;
import com.mapreduce.wordcount.WordCountMapper;
import com.mapreduce.wordcount.WordCountReducer;
import com.mapreduce.wordcount.WordCountRunJob;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import java.io.IOException;
import java.net.URI;
public class SortApp {
private static final String INPUT_PATH = "hdfs://master001:9000/sort";
private static final String OUTPUT_PATH = "hdfs://master001:9000/outputsort";
public static class MyMapper extends Mapper<LongWritable, Text, IntWritable, IntWritable>{
private static IntWritable data = new IntWritable(); //静态变量了解一下
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException{
String line = value.toString();
data.set(Integer.parseInt(line));
context.write(data, new IntWritable(1));
}
}
public static class MyReducer extends Reducer<IntWritable, IntWritable, IntWritable, IntWritable>{
private static IntWritable data = new IntWritable(1);
public void reduce(IntWritable key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException{
for(IntWritable val : values){
context.write(data, key);
data = new IntWritable(data.get() + 1);
}
}
}
public static void main(String[] args) throws Exception{
System.setProperty("HADOOP_USER_NAME", "hadoop");
Configuration conf = new Configuration();
//提升代码的健壮性
final FileSystem fileSystem = FileSystem.get(URI.create(INPUT_PATH), conf);
if(fileSystem.exists(new Path(OUTPUT_PATH))){
fileSystem.delete(new Path(OUTPUT_PATH), true);
}
Job job = Job.getInstance(conf, "SortApp");
//run jar class 主方法
job.setJarByClass(SortApp.class);
//设置map
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(IntWritable.class);
//设置reduce
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(IntWritable.class);
//设置input format
job.setInputFormatClass(TextInputFormat.class);
FileInputFormat.addInputPath(job, new Path(INPUT_PATH));
//设置output format
job.setOutputFormatClass(TextOutputFormat.class);
FileOutputFormat.setOutputPath(job, new Path(OUTPUT_PATH));
//提交job
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}