电话号码分区案例

需求：要求将统计结果按照条件输出到不同文件中（分区）。比如：将统计结果按照手机号不同的运营商输出到不同文件中（分区）

 

数据样本模式：一共180万个电话号码左右

 

 

这个是Mapl类代码：

package com.itstar.mr.wc0908.Partition;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class PartitionMapper extends Mapper<LongWritable,Text,Text, NullWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //1.获取数据
        String line = value.toString();
        //2.切分数据“\t”
        String[] phions = line.split("\t");
        for (String phion:phions) {
            context.write(new Text(phion),NullWritable.get());
        }


    }
}

 

Partition类：

 

package com.itstar.mr.wc0908.Partition;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;

import java.util.Arrays;

public class PhionPartition extends Partitioner<Text, NullWritable> {
    //移动号码前三位
    public static final String[] YD = {
            "134", "135", "136",
            "137", "138", "139",
            "150", "151", "152",
            "157", "158", "159",
            "188", "187", "182",
            "183", "184", "178",
            "147", "172", "198"
    };
    //电信号码前三位
    public static final String[] DX = {
            "133", "149", "153",
            "173", "177", "180",
            "181", "189", "199"};
    //联通号码前三位
    public static final String[] LT = {
            "130", "131", "132",
            "145", "155", "156",
            "166", "171", "175",
            "176", "185", "186", "166"
    };
    @Override
    public int getPartition(Text text, NullWritable nullWritable, int i) {

        String phione = text.toString();
        String phione1 = phione.substring(0, 3);
        //判断手机号前三位属于哪个运营商
        if (Arrays.asList(YD).contains(phione1)) {
            return 0;
        } else if (Arrays.asList(LT).contains(phione1)) {
            return 1;
        } else {
            return 2;
        }
    }
}

 

reduce类：

 

package com.itstar.mr.wc0908.Partition;

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;


public class PartitionReduce extends Reducer<Text,NullWritable,Text, NullWritable> {

    @Override
    protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException, IOException {
        context.write(key,NullWritable.get());
    }
}

 

Drive类：

 

 

package com.itstar.mr.wc0908.Partition;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


import java.io.IOException;

public class PartitionDrive {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

        //实例化配置文件
        Configuration configuration = new Configuration();

        //定义一个job任务
        Job job = Job.getInstance(configuration);

        //配置job的信息
         job.setJarByClass(PartitionDrive.class);

         //配置map的信息
        job.setMapperClass(PartitionMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        //设置map的预合并
        job.setPartitionerClass(PhionPartition.class);

        //设置reduce的信息
        job.setReducerClass(PartitionReduce.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(NullWritable.class);

        //自定义分区，判断手机号的前三位属于哪个运营商来分区
        job.setPartitionerClass(PhionPartition.class);
        job.setNumReduceTasks(3);

        //配置输入数据的路径  查看继承树的快捷键 Ctrl+h
        FileInputFormat.setInputPaths(job,new Path("D:\\大数据中高期\\08-16shuffle\\phone.txt"));
        //配置输出数据的路径
        FileOutputFormat.setOutputPath(job,new Path("D:\\大数据中高期\\08-16shuffle\\outphone"));


        //提交job任务
        job.waitForCompletion(true);

    }
}

 

运行成功的结果：

 

 

下面的是移动电话号码生成的:

 

 

下面的是联通电话号码生成的:

 

 

下面的是电信电话号码生成的:

 

 

 

注意：

如果reduceTask的数量> getPartition的结果数，则会多产生几个空的输出文件part-r-000xx；

如果1<reduceTask的数量<getPartition的结果数，则有一部分分区数据无处安放，会Exception；

如果reduceTask的数量=1，则不管mapTask端输出多少个分区文件，最终结果都交给这一个reduceTask，最终也就只会产生一个结果文件 part-r-00000；

 

如果：3个分区对应的是一个，设置job.setNumReduceTasks(4)是4则，有一个文件是空的

                                             ，设置job.setNumReduceTasks(2)是2则，则会报错

                                           ，设置job.setNumReduceTasks(1)是1则，不会报错，则只是会生成一个文件