Hadoop之MapReduce （Partitioner）

今天我们用Partitioner按电话号查找需要字段的加和。

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.log4j.BasicConfigurator;

import javax.tools.ForwardingJavaFileObject;
import java.io.IOException;

public class FlowSum {
    public static class FlowSumMap extends Mapper<LongWritable, Text,Text, IntWritable>{
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] split = value.toString().split("\t");
            //取出电话号
            String tellNum=split[1];
            //取出需要得数据，，加和
            Integer upFlow = Integer.parseInt(split[8]);
            Integer downFlow = Integer.parseInt(split[9]);
            Integer sumFlow = upFlow + downFlow;
            context.write(new Text(tellNum),new IntWritable(sumFlow));
        }
    }
    public static class FlowSumReduce extends Reducer<Text,IntWritable,Text,IntWritable>{
        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            Integer heFlow = 0;
            for (IntWritable value : values) {
                //重复的电话号也会加起来，
                heFlow += value.get();
            }
            context.write(key,new IntWritable(heFlow));
        }
    }

    public static void main(String[] args) throws Exception {
        BasicConfigurator.configure();//自动快速使用缺省log4j的环境
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        job.setJarByClass(FlowSum.class);
        //里面参数是分区，写 1 是默认值 不分区。都在一个里。不能少于你所定义的分区个数。
        job.setNumReduceTasks(8);
        //具体使用了哪个partition
        job.setPartitionerClass(PrivatePatitioner.class);


        job.setMapperClass(FlowSumMap.class);
        job.setReducerClass(FlowSumReduce.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        FileInputFormat.setInputPaths(job,new Path("D:\\eclipse\\wc\\input\\flow.log"));
        FileOutputFormat.setOutputPath(job,new Path("D:\\eclipse\\wc\\outputflow"));

        job.submit();
        boolean b = job.waitForCompletion(true);
        System.exit(b?0:1);
    }
}

Partitioner类

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;

import java.util.HashMap;

public class PrivatePatitioner extends Partitioner<Text, IntWritable> {
    private static HashMap<String,Integer> privateMap = new HashMap<>();
    static {
        privateMap.put("135",0);
        privateMap.put("136",1);
        privateMap.put("137",2);
        privateMap.put("138",3);
        privateMap.put("139",4);
    }
    @Override
    public int getPartition(Text text, IntWritable intWritable, int i) {
        String tellNum = text.toString().substring(0,3);
        Integer integer = privateMap.get(tellNum);
        //如果integer 没有值 说明不在上面的字段中。除了上面的字段，其他的都放在5
        return integer == null ? 5 : integer;
    }
}

结果

内容