招聘网站—MapReduce数据清洗

Wa_Automata

已于 2023-11-04 15:56:57 修改

阅读量494

点赞数

分类专栏：头歌实践教学平台文章标签： mapreduce hadoop 大数据

于 2023-05-31 17:03:31 首次发布

本文链接：https://blog.csdn.net/qq_52792570/article/details/130973065

版权

头歌实践教学平台专栏收录该内容

9 篇文章 12 订阅

订阅专栏

招聘网站—MapReduce数据清洗

第1关：数据清洗

package recruit.mapreduce;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class RecruitMap extends Mapper<LongWritable, Text, Text, NullWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        /********** Begin **********/
        String[] fields = value.toString().split("\t");
        boolean flag = true;
        if (fields.length == 9) {
            int i = 0;
            for (String field : fields) {
                if ("".equals(field) || field.trim() == null) {
                    flag = false;
                    break;
                }
            }
            if (flag) {
                if (fields[2].contains("k")) {
                    fields[2] = fields[2].replace("k", "");
                } else if (fields[2].contains("K")) {
                    fields[2] = fields[2].replace("K", "");
                }
                String re = "[0-9]+-[0-9]+";
                flag = fields[2].matches(re);
            }
            if (flag) {
                fields[0] = fields[0].toLowerCase();
                fields[8] = fields[8].toLowerCase();
                fields[8] = fields[8].replaceAll("·", "|");
                fields[1] = fields[1].trim().split("·")[0].trim();
                String maxsalary = fields[2].split("-")[0].trim();
                String minsalary = fields[2].split("-")[1].trim();
                Double avgsalary = (Double.parseDouble(maxsalary) + Double.parseDouble(minsalary)) / 2;
                fields[2] = String.format("%.2f", avgsalary);
                String result = "";
                int len = 0;
                for (String field : fields) {
                    len = len + 1;
                    if (len == fields.length) {
                        result = result + field;
                    } else {
                        result = result + field + "\t";
                    }
                }
                context.write(new Text(result), NullWritable.get());
            }
        }
        /********** End **********/
    }
}

package recruit.mapreduce;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class RecruitReduce extends Reducer<Text, NullWritable, NullWritable, Text> {
    @Override
    protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
        /********** Begin **********/
        context.write(NullWritable.get(), new Text(key));
        /********** End **********/
    }
}

package recruit.mapreduce;


import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class RecruitJob {
    public static void main(String[] args) throws Exception {
        /********** Begin **********/
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJarByClass(RecruitJob.class);
        job.setMapperClass(RecruitMap.class);
        job.setReducerClass(RecruitReduce.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);
        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(Text.class);
        String outputpath = "/root/files";
        Path path = new Path(outputpath);
        FileSystem fileSystem = path.getFileSystem(conf);
        if (fileSystem.exists(path)) {
            fileSystem.delete(path, true);
        }
        FileInputFormat.addInputPath(job, new Path("/data/workspace/myshixun/clean/data/advertise.txt"));
        FileOutputFormat.setOutputPath(job, path);
        job.waitForCompletion(true);

        /********** End **********/
    }
}