测试数据:
file1.txt
2
32
654
32
15
756
65223
file2.txt
5956
22
650
92
file3.txt
26
54
6
Mapper代码:
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class MyMapper extends Mapper<LongWritable, Text, IntWritable, NullWritable> {
//private Text val = new Text("");
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//context.write(new IntWritable(Integer.valueOf(value.toString().trim())), val);
context.write(new IntWritable(Integer.valueOf(value.toString().trim())), NullWritable.get());
}
}
reduce代码:
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class MyReducer extends Reducer<IntWritable, NullWritable, IntWritable, IntWritable> {
private IntWritable order = new IntWritable(1);
@Override
protected void reduce(IntWritable key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
for(NullWritable val : values) {
context.write(order, key);
order = new IntWritable(order.get() + 1);
}
}
}
main代码:
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
/**
* 把输入的数据,按照升序排序,并且每一行都有序号
* 并且,相同的数据不能被去重(关键点是在reduce的for循环,如果不写for循环,则会去重)
*/
public class JobMain {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
if(args.length != 2) {
System.err.println("Usage: SortDataExample<input path> <output path>");
System.exit(-1);
}
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "SortJob");
job.setJarByClass(JobMain.class);
job.setMapperClass(MyMapper.class);
job.setMapOutputKeyClass(IntWritable.class);
//job.setMapOutputValueClass(Text.class);
job.setMapOutputValueClass(NullWritable.class);
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(IntWritable.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
Path path = new Path(args[1]);
FileSystem fs = FileSystem.get(conf);
if(fs.exists(path)) {
fs.delete(path, true);
}
FileOutputFormat.setOutputPath(job, path);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
运行代码
把3个数据测试文件放到路径:C:…\mr_example\exp_03\下
在windows7环境下执行前需要设置HADOOP_CLASSPATH
set HADOOP_CLASSPATH=mr_example.jar
注:mr_example.jar是以上代码打成jar包的名称
执行命令
hadoop jar的全类名(包名+类名) 数据路径 输出路径
eg:
C:\hadoop-2.9.1\bin\hadoop jar的全类名(包名+类名) file:///C:…\mr_example\exp_03\*.txt C:…\mr_example\exp_03\out\
结果:
1 2
2 6
3 15
4 22
5 26
6 32
7 32
8 54
9 92
10 650
11 654
12 756
13 5956
14 65223