MapReduce去重复数据打包服务器运行随堂笔记202111

一、 MapReduce去重复数据

案例：创建一个文件 1.txt
A 1
a 2
b 23
a 2
c 34
输出结果：
A 1
a 2
b 23
c 34

二、 准备工作

1. 1.txt
2. 将1.txt(本地文件)上传到hdfs文件系统中【hafs dfs –put 】
3. 将idea程序打jar包
4. 执行hadoop jar包的命令【hadoop jar jar_name class_name hdfs文件 输出路径】

三、 细节

1. Mavan 本地仓库
   

2. Jar包

四、 代码

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class DuplicationMapReduce {
  /*
     MyMapper class  extends Mapper
     MyReducer class  extends Reducer

    */
    public static class MyMapper extends Mapper<LongWritable, Text,Text,Text> {
      private static Text line = new Text();
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
     /* super.map(key, value, context);*/
      line = value;
      context.write(line,new Text(""));
    }
  }
    public static class MyReducer extends Reducer<Text,Text,Text,Text> {
      @Override
      protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
        /*super.reduce(key, values, context);*/
        context.write(key,new Text(""));
      }
    }
    public static void main(String[] args) throws IOException{
      Job job = Job.getInstance(new Configuration());
      job.setJarByClass(DuplicationMapReduce.class);// className
      job.setMapperClass(MyMapper.class);
      job.setReducerClass(MyReducer.class);
      job.setOutputKeyClass(Text.class);
      job.setOutputValueClass(Text.class);
      FileInputFormat.setInputPaths(job,new Path(args[0]));
      FileOutputFormat.setOutputPath(job,new Path(args[1]));
      try{
        Boolean result = job.waitForCompletion(true);
        System.exit(result?0:1);
      }catch(Exception e){
        e.printStackTrace();
      }
    }
}

五 、上传jar 包

六、 测试

hdfs dfs –put 1.txt /

hadoop jar Projectwx-1.0-SNAPSHOT.jar DuplicationMapReduce /1.txt /2021

hdfs dfs -cat /2021/part-r-00000