一、 MapReduce去重复数据
案例:创建一个文件 1.txt
A 1
a 2
b 23
a 2
c 34
输出结果:
A 1
a 2
b 23
c 34
二、 准备工作
- 1.txt
- 将1.txt(本地文件)上传到hdfs文件系统中【hafs dfs –put 】
- 将idea程序打jar包
- 执行hadoop jar包的命令【hadoop jar jar_name class_name hdfs文件 输出路径】
三、 细节
-
Mavan 本地仓库
-
Jar包
四、 代码
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class DuplicationMapReduce {
/*
MyMapper class extends Mapper
MyReducer class extends Reducer
*/
public static class MyMapper extends Mapper<LongWritable, Text,Text,Text> {
private static Text line = new Text();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
/* super.map(key, value, context);*/
line = value;
context.write(line,new Text(""));
}
}
public static class MyReducer extends Reducer<Text,Text,Text,Text> {
@Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
/*super.reduce(key, values, context);*/
context.write(key,new Text(""));
}
}
public static void main(String[] args) throws IOException{
Job job = Job.getInstance(new Configuration());
job.setJarByClass(DuplicationMapReduce.class);// className
job.setMapperClass(MyMapper.class);
job.setReducerClass(MyReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));
try{
Boolean result = job.waitForCompletion(true);
System.exit(result?0:1);
}catch(Exception e){
e.printStackTrace();
}
}
}
五 、上传jar 包
六、 测试
hdfs dfs –put 1.txt /
hadoop jar Projectwx-1.0-SNAPSHOT.jar DuplicationMapReduce /1.txt /2021
hdfs dfs -cat /2021/part-r-00000