_mapreducce计算手机流量使用

FlowDriver.java

package com.igeekhome.mapreduce.flow;

import com.igeekhome.mapreduce.model.FlowBean;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class FlowDriver  {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {


        //1.获取配置信息对象和job对象
        Configuration conf=new Configuration();
        Job job=Job.getInstance(conf);

        //2.关联Driver类
        job.setJarByClass(FlowDriver.class);

        //3.设置mapper和reducer对象
        job.setMapperClass(FlowMapper.class);
        job.setReducerClass(FlowReducer.class);

        //4.设置mapper输出的kv类型
        job.setMapOutputValueClass(LongWritable.class);
        job.setMapOutputValueClass(FlowBean.class);

        //5.设置最终输出的kv类型（reducer输出的kv类型）
        job.setOutputKeyClass(LongWritable.class);
        job.setOutputValueClass(FlowBean.class);

        //6.设置文件的输入路径和计算结果的输出路径
        FileInputFormat.setInputPaths(job,new Path(args[0]));

        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        //7.设置自定义分区
        job.setPartitionerClass(PhoneNumberPartitioner.class);

        //设置Reduce Task的个数
        job.setNumReduceTasks(3);

        //8.提交任务 进行计算
        boolean result=job.waitForCompletion(true);


        System.out.println(result?"执行成功":"执行失败");
    }
}

 FlowMapper.java

package com.igeekhome.mapreduce.flow;


import com.igeekhome.mapreduce.model.FlowBean;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class FlowMapper extends Mapper<LongWritable, Text,LongWritable, FlowBean> {

    //新建FlowBean对象 作为输出的value
    private FlowBean valueOut=new FlowBean();

    //新建LongWritable作为输出的key
    private LongWritable keyOut=new LongWritable();

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //获取一行数据
        String line=value.toString();

        //根据分隔符进行数据拆分
        String[] flowData=line.split(",");

        //根据下标提取数据
        String phoneNumber=flowData[0];
        String upFlow=flowData[1];
        String downFlow=flowData[2];

        //为对象赋值
        valueOut.setPhoneNumber(Long.parseLong(phoneNumber.trim()));
        valueOut.setUpFlow(Long.parseLong(upFlow.trim()));
        valueOut.setDownFlow(Long.parseLong(downFlow.trim()));

        //计算总量
        valueOut.setSumFlow();

        //为输出的key赋值
        keyOut.set(Long.parseLong(phoneNumber.trim()));

        //map阶段输出
        context.write(keyOut,valueOut);
    }
}

FlowReducer.java 

package com.igeekhome.mapreduce.flow;

import com.igeekhome.mapreduce.model.FlowBean;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class FlowReducer extends Reducer<LongWritable, FlowBean,LongWritable, FlowBean> {

    //创建FlowBean对象，作为输出的value
    private FlowBean valueOut=new FlowBean();
    @Override
    protected void reduce(LongWritable key, Iterable<FlowBean> values, Context context) throws IOException, InterruptedException {

        //定义一个手机号码的全部流量
        Long totalSumFlow=0L;

        //累加计算总流量
        for (FlowBean flowBean : values) {
            Long sumFlow=flowBean.getSumFlow();

            totalSumFlow+=sumFlow;
        }

        //为输出的value赋值
        valueOut.setSumFlow(totalSumFlow);

        //reduce阶段输出
        context.write(key,valueOut);

    }
}

 PhoneNumberPartitioner.java

package com.igeekhome.mapreduce.flow;

import com.igeekhome.mapreduce.model.FlowBean;
import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.mapreduce.Partitioner;

public class PhoneNumberPartitioner extends Partitioner<LongWritable, FlowBean> {

    @Override
    public int getPartition(LongWritable longWritable, FlowBean flowBean, int numberPartitioner) {
        //获取手机号码的前三位
        String phoneSubstr=flowBean.getPhoneNumber().toString().substring(0,3);

        //定义分区号
        int partitionNum=0;

        if(phoneSubstr.equals("136")){
            partitionNum=0;
        }else if(phoneSubstr.equals("138")){
            partitionNum=1;
        }else {
            partitionNum=2;
        }


        return partitionNum;
    }
}

 FlowBean.java

package com.igeekhome.mapreduce.model;

import org.apache.hadoop.io.Writable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

//手机流量模型
public class FlowBean implements Writable {
    //手机号码

    private Long phoneNumber;

    //上行流量
    private Long upFlow;

    //下行流量
    private Long downFlow;

    //总流量（上行＋下行流量）
    private Long sumFlow;



    public FlowBean() {
    }

    public FlowBean(long phoneNumber, long upFlow, long downFlow) {
        this.phoneNumber = phoneNumber;
        this.upFlow = upFlow;
        this.downFlow = downFlow;
    }

    //重写序列化方法
    @Override
    public void write(DataOutput out) throws IOException {

        out.writeLong(phoneNumber);
        out.writeLong(upFlow);
        out.writeLong(downFlow);
        out.writeLong(sumFlow);

    }

    //重写反序列化方法
    @Override
    public void readFields(DataInput in) throws IOException {
        this.phoneNumber=in.readLong();
        this.upFlow=in.readLong();
        this.downFlow=in.readLong();
        this.sumFlow=in.readLong();

    }

    @Override
    public String toString() {
        return this.sumFlow.toString();
    }

    public Long getPhoneNumber() {
        return phoneNumber;
    }

    public void setPhoneNumber(Long phoneNumber) {
        this.phoneNumber = phoneNumber;
    }

    public Long getUpFlow() {
        return upFlow;
    }

    public void setUpFlow(Long upFlow) {
        this.upFlow = upFlow;
    }

    public Long getDownFlow() {
        return downFlow;
    }

    public void setDownFlow(Long downFlow) {
        this.downFlow = downFlow;
    }

    public Long getSumFlow() {
        return sumFlow;
    }

    public void setSumFlow(Long sumFlow) {
        this.sumFlow = sumFlow;
    }
    //计算总流量
    public void setSumFlow() {
        this.sumFlow = this.upFlow+this.downFlow;
    }
}

 

phone_data.txt
-----------------------
13800138001,1024,2048
13900139002,512,1024  
13800138003,256,512
13700137004,768,1536  
13600136005,128,256 
13800138001,1536,3072  
13900139002,2048,4096  
13700137003,512,1024 
13600136004,768,1536  
13800138005,128,256 
13900139001,384,768
13700137002,1024,2048 
13600136005,1536,3072
13800138003,2048,4096
13900139004,512,1024 
13700137001,768,1536
13600136002,128,256
13800138006,384,768
13900139005,1024,2048
13700137006,1536,3072  
13600136003,2048,4096  
13800138002,512,1024  
13900139003,768,1536  
13700137005,128,256  
13600136001,384,768  
13800138004,1024,2048  
13900139006,1536,3072  
13700137007,2048,4096  
13600136007,512,1024  
13800138007,768,1536  
13900139007,128,256  
13700137008,384,768  
13600136008,1024,2048  
13800138008,1536,3072  
13900139008,2048,4096  
13700137009,512,1024  
13600136009,768,1536  
13800138009,128,256  
13900139009,384,768  
13700137010,1024,2048  
13600136010,1536,3072  
13800138010,2048,4096  
13900139010,512,1024  
13700137011,768,1536  
13600136011,128,256  
13800138011,384,768  
13900139011,1024,204

 程序运行成功后，开启虚拟机dfs、yarn服务，打包mapreduce_demo.jar放到/opt/jar文件夹中，将文件phone_data.txt上传到/input文件夹

 

复制执行程序的全路径

hadoop jar jar包 执行程序的全路径 文件输入路径 文件输出路径
eg:
hadoop jar /opt/jar/mapreduce_demo.jar com.igeekhome.mapredu                                                                               ce.flow.FlowDriver /input /output

注意：确保输出路径不提前存在

part-r-00000

part-r-00001

part-r-00002