【Hadoop学习项目】2. 数据去重

辰阳星宇

已于 2022-02-03 13:08:44 修改

阅读量1.8k

点赞数 1

分类专栏： # Hadoop 文章标签： hadoop mapreduce big data

于 2022-02-01 20:08:44 首次发布

本文链接：https://blog.csdn.net/qq_41094332/article/details/122765278

版权

Hadoop 专栏收录该内容

20 篇文章 4 订阅

订阅专栏

该博客介绍了一个基于Hadoop实现数据去重的项目。`DupDriver`类负责配置和启动作业，`DupMapper`将输入数据的每一行作为键输出，`DupReducer`则用于收集相同的键并输出，实现数据去重。通过这个过程，可以高效地处理大量重复IP地址的数据。

摘要由CSDN通过智能技术生成

0. 项目结构

在这里插入图片描述
数据处理过程图

1. DupDriver

package hadoop_test.data_duplicate_demo_02;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class DupDriver {
/* 处理数据：
192.168.234.21
192.168.234.22
192.168.234.21
192.168.234.21
192.168.234.23
192.168.234.21
192.168.234.21
192.168.234.21
192.168.234.25
192.168.234.21
192.168.234.21
192.168.234.26
192.168.234.21
192.168.234.27
192.168.234.21
192.168.234.27
192.168.234.21
192.168.234.29
192.168.234.21
192.168.234.26
192.168.234.21
192.168.234.25
192.168.234.25
192.168.234.21
192.168.234.22
192.168.234.21
 */
public static void main(String[] args) throws Exception {

    System.setProperty("HADOOP_USER_NAME", "root");

    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf);

    job.setJarByClass(DupDriver.class);

    job.setMapperClass(DupMapper.class);
    job.setReducerClass(DupReducer .class);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(NullWritable.class);	// Value为NULL，具体原因后续会有说明
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(NullWritable.class);

    FileInputFormat.setInputPaths(job, new Path("/hadoop_test/dup/dup.txt"));
    FileOutputFormat.setOutputPath(job, new Path("/hadoop_test/dup/word_count_result"));
    job.waitForCompletion(true);
}
}

2. DupMapper

package hadoop_test.data_duplicate_demo_02;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class DupMapper extends Mapper<LongWritable,Text,Text,NullWritable> {

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//      value : 192.168.70.49
//      因为目标为去重，不用统计数量。因此不用让key为1，可减少IO时间开销
        context.write(new Text(value),NullWritable.get());
    }
}

3. DupReducer

package hadoop_test.data_duplicate_demo_02;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class DupReducer  extends Reducer<Text,NullWritable,Text,NullWritable> {

    @Override
    protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
        context.write(new Text(key),NullWritable.get());
    }
}