Hadoop Partitioner的应用

最新推荐文章于 2022-10-27 18:04:02 发布

touero

最新推荐文章于 2022-10-27 18:04:02 发布

阅读量357

点赞数

文章标签： hadoop 大数据分布式

本文链接：https://blog.csdn.net/Tlouer_Elle/article/details/127342875

版权

读取数据，没什么特别的，按行读取，作为（key，value）中的key，vaule为null


package mapper;

import java.io.IOException;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class MyMapper extends Mapper<Object, Text, Text, NullWritable> {
    private Text ks = new Text();

    public MyMapper() {
    }

    public void map(Object key, Text value, Mapper<Object, Text, Text, NullWritable>.Context context) throws IOException, InterruptedException {
        this.ks.set(value.toString());
        context.write(this.ks, NullWritable.get());
    }
}

Partitioner组件可以让Map对Key进行分区，从而可以根据不同的key分发到不同的Reduce中去处理，其目的就是将key均匀分布在ReduceTask上。

//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by FernFlower decompiler)
//

package partitioner;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;

public class MyPartitioner extends Partitioner<Text, NullWritable> {
    public MyPartitioner() {
    }

    public int getPartition(Text key, NullWritable nullWritable, int i) {
        String[] st = key.toString().split("\t");
        String usercounts = st[2];
        int usercount = Integer.parseInt(usercounts);
        String useraveragecash = st[1];
        int useraveragecashs = Integer.parseInt(useraveragecash);
        if (usercount > 6) {
            return useraveragecashs > 106 ? 1 : 2;
        } else if (usercount < 6) {
            return useraveragecashs > 106 ? 3 : 4;
        } else {
            return 0;
        }
    }
}

reduce汇总也没什么特别的

//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by FernFlower decompiler)
//

package reducer;

import java.io.IOException;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class MyReducer extends Reducer<Text, NullWritable, Text, NullWritable> {
    public MyReducer() {
    }

    public void reduce(Text key, Iterable<NullWritable> values, Reducer<Text, NullWritable, Text, NullWritable>.Context context) throws IOException, InterruptedException {
        context.write(key, NullWritable.get());
    }
}

Job设置

//
// Source code recreated from a .class file by IntelliJ IDEA
// (powered by FernFlower decompiler)
//

package job;

import java.io.IOException;
import mapper.MyMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import partitioner.MyPartitioner;
import reducer.MyReducer;

public class MyJob {
    public MyJob() {
    }

    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        Job job = Job.getInstance(new Configuration());
        job.setJarByClass(MyJob.class);
        job.setMapperClass(MyMapper.class);
        job.setReducerClass(MyReducer.class);
        job.setPartitionerClass(MyPartitioner.class);
        job.setNumReduceTasks(5);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(NullWritable.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);
        job.setJobName("myResult");
        FileInputFormat.setInputPaths(job, "D:\\hadoop\\output\\clearUser\\part-r-00000");
        FileOutputFormat.setOutputPath(job, new Path("D:\\hadoop\\output\\classify"));
        boolean flag = job.waitForCompletion(true);
        System.exit(flag ? 0 : 1);
    }
}