MapReduce——手机流量分析

现有手机流量日志，记录有手机号码、MAC地址、IP地址、浏览内容、上行流量、下行流量等，

A.分析每个手机号的上行流量、下行流量和总流量（需要一个流量实体）

B.将结果按总流量排序，总流量相同按下行流量排序

C.求流量排行前三的手机号

一、Flow实体

import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/*
自定义数据类型时一定要注意，要是想在Map或Reduce阶段输出
①做值必须实现Writable接口
②做键必须实现WritableComparable接口
③正确重写序列化方法
④正确重写比较方法
 */

public class Flow implements WritableComparable<Flow>{
    private int upFlow;
    private int downFlow;
    private int sumFlow;
    private String phoneNumber="";//必须赋初值，防止序列化失败
    public Flow(){

    }
    public Flow (Flow f){
        this.phoneNumber=f.phoneNumber;
        this.upFlow=f.upFlow;
        this.downFlow=f.downFlow;
        this.sumFlow=f.sumFlow;
    }
    public Integer getUpFlow() {
        return upFlow;
    }

    public void setUpFlow(Integer upFlow) {
        this.upFlow = upFlow;
    }

    public Integer getDownFlow() {
        return downFlow;
    }

    public void setDownFlow(Integer downFlow) {
        this.downFlow = downFlow;
    }

    public Integer getSumFlow() {
        this.sumFlow=this.upFlow+this.downFlow;
        return sumFlow;
    }

    public void setSumFlow(Integer sumFlow) {
        this.sumFlow = sumFlow;
    }

    public String getPhoneNumber() {
        return phoneNumber;
    }

    public void setPhoneNumber(String phoneNumber) {
        this.phoneNumber = phoneNumber;
    }

    @Override
    public String toString() {
        return phoneNumber+"\t"+upFlow+"\t"+downFlow+"\t"+sumFlow;
    }

    public int compareTo(Flow o) {
        if(this.sumFlow==o.sumFlow){
            return o.downFlow-this.downFlow;
        }else {
            return o.sumFlow-this.sumFlow;
        }
    }
    /*
    重写序列化方法时读写顺序要一致！！！类型要一致！！！
     */
    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeInt(upFlow);
        dataOutput.writeInt(downFlow);
        dataOutput.writeInt(sumFlow);
        dataOutput.writeUTF(phoneNumber);
    }

    public void readFields(DataInput dataInput) throws IOException {
        upFlow=dataInput.readInt();
        downFlow=dataInput.readInt();
        sumFlow=dataInput.readInt();
        phoneNumber=dataInput.readUTF();
    }
}

二、分析每个手机的上行流量、下行流量和总流量

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

/*
分析每个手机的上行流量、下行流量和总流量
需要自己制作一个流量实体
 */
public class ForPhoneFlow {
        public static class ForMapper extends Mapper<LongWritable,Text,Text,Flow> {
            @Override
            protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
                String line=value.toString();
                Flow flow=new Flow();
                String []strs=line.split("\t");
                flow.setPhoneNumber(strs[1]);
                flow.setUpFlow(Integer.parseInt(strs[8]));
                flow.setDownFlow(Integer.parseInt(strs[9]));
                flow.getSumFlow();
                context.write(new Text(strs[1]),flow);
            }
        }
        public static class ForReducer extends Reducer<Text,Flow,Flow,NullWritable> {
            @Override
            protected void reduce(Text key, Iterable<Flow> values, Context context) throws IOException, InterruptedException {
                int sumUp=0;
                int sumDown=0;
                for(Flow f:values){
                    sumUp+=f.getUpFlow();
                    sumDown+=f.getDownFlow();
                }
                Flow flow=new Flow();
                flow.setPhoneNumber(key.toString());
                flow.setUpFlow(sumUp);
                flow.setDownFlow(sumDown);
                flow.getSumFlow();
                context.write(flow,NullWritable.get());
            }
        }

    public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException, URISyntaxException {
        Job job= Job.getInstance();
        job.setMapperClass(ForMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Flow.class);
        job.setReducerClass(ForReducer.class);
        job.setOutputKeyClass(Flow.class);
        job.setOutputValueClass(NullWritable.class);
        FileInputFormat.addInputPath(job,new Path("E://forTestData//phoneFlow//data"));
        FileSystem fileSystem=FileSystem.get(new URI("file:e//output"),new Configuration());
        Path path=new Path("e://output");
        if(fileSystem.exists(path)){
            fileSystem.delete(path,true);
        }
        FileOutputFormat.setOutputPath(job,path);
        job.setNumReduceTasks(1);//若要求出全局TopN，map处理完的数据只能交付给一个reduce
        job.waitForCompletion(true);
    }
}
/*
中间还很出现了两个小错误：
①Unable to initialize any output collector
注意一下通过Job给Map或Reduce设置的输出类型和写类的时候的那个泛型是否一致！
因为我中间改过输出类型，只改了上面没改下面
②Unable to initialize any output collector
是因为输出类型是自定义的，但是我没有实现WritableComparable接口
 */

三、按用户的总流量排序，总流量相同按下行流量排序

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
/*
分析每个手机的上行流量、下行流量和总流量
并按照总流量排序
如果总流量相同，按下行流量排序
 */
public class ForSortPhoneFlow {
    public static class ForMapper extends Mapper<LongWritable,Text,Text,Flow>{
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line=value.toString();
            String []strs=line.split("\t");
            Flow flow=new Flow();
            flow.setPhoneNumber(strs[1]);
            flow.setUpFlow(Integer.parseInt(strs[8]));
            flow.setDownFlow(Integer.parseInt(strs[9]));
            flow.getSumFlow();
            context.write(new Text(strs[1]),flow);
        }
    }
    public static class ForReducer extends Reducer<Text,Flow,Flow,NullWritable>{
        List<Flow> list=new ArrayList<Flow>();
        @Override
        protected void reduce(Text key, Iterable<Flow> values, Context context) throws IOException, InterruptedException {
            int sumUp = 0;
            int sumDown = 0;
            for (Flow f : values) {
                sumUp += f.getUpFlow();
                sumDown += f.getDownFlow();
            }
            Flow f = new Flow();
            f.setPhoneNumber(key.toString());
            f.setUpFlow(sumUp);
            f.setDownFlow(sumDown);
            f.getSumFlow();
            list.add(f);
        }

        @Override
        protected void cleanup(Context context) throws IOException, InterruptedException {
            Collections.sort(list, new Comparator<Flow>() {
                public int compare(Flow o1, Flow o2) {
                    if(o1.getSumFlow()==o2.getSumFlow()){
                        return o2.getDownFlow()-o1.getDownFlow();
                    }else {
                        return o2.getSumFlow()-o1.getSumFlow();
                    }
                }
            });
            for(Flow f:list){
                context.write(f, NullWritable.get());
            }
        }
    }
    public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException {
        Job job= Job.getInstance();
        job.setMapperClass(ForMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Flow.class);
        job.setReducerClass(ForReducer.class);
        job.setOutputKeyClass(Flow.class);
        job.setOutputValueClass(NullWritable.class);

        FileSystem fileSystem=FileSystem.get(new URI("file:E://output"),new Configuration());
        Path path=new Path("E://output");
        if(fileSystem.exists(path)){
            fileSystem.delete(path,true);
        }
        FileInputFormat.addInputPath(job,new Path("E://forTestData//phoneFlow//data"));
        FileOutputFormat.setOutputPath(job,path);
        job.setNumReduceTasks(1);
        job.waitForCompletion(true);
    }
}

四、在（二）的输出结果的基础上，计算手机流量排行前三的手机号

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

/*
在ForPhoneFlow的基础上
计算手机流量排行前三的手机号
 */
public class ForTopThreePhone {
    public static class ForMapper extends Mapper<LongWritable,Text,Text,Flow>{
        List<Flow> list=new ArrayList<Flow>();
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line=value.toString();
            String []strs=line.split("\t");
            Flow flow=new Flow();
            flow.setPhoneNumber(strs[0]);
            flow.setUpFlow(Integer.parseInt(strs[1]));
            flow.setDownFlow(Integer.parseInt(strs[2]));
            flow.getSumFlow();
            list.add(flow);
            if(list.size()==4){
                Collections.sort(list);
                list.remove(3);
            }
        }

        @Override
        protected void cleanup(Context context) throws IOException, InterruptedException {
            for(int i=0;i<3;i++){
                context.write(new Text(list.get(i).getPhoneNumber()),list.get(i));
            }
        }
    }
    public static class ForReducer extends Reducer<Text,Flow,Flow,NullWritable>{
        List<Flow> list=new ArrayList<Flow>();
        @Override
        protected void reduce(Text key, Iterable<Flow> values, Context context) throws IOException, InterruptedException {
            /*
            往List集合里面添加的时候，一定要新建一个对象
            不然加到List中的都是同一个引用，List里的存的对象都是同一个对象
             */
            list.add(new Flow(values.iterator().next()));

        }
        @Override
        protected void cleanup(Context context) throws IOException, InterruptedException {
            Collections.sort(list);
            for(int i=0;i<3;i++){
                context.write(list.get(i),NullWritable.get());
            }
        }
    }

    public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException {
        Job job=Job.getInstance();
        job.setMapperClass(ForMapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(Flow.class);
        job.setReducerClass(ForReducer.class);
        job.setOutputKeyClass(Flow.class);
        job.setOutputValueClass(NullWritable.class);
        FileSystem fileSystem=FileSystem.get(new URI("file:E://output"),new Configuration());
        Path path=new Path("E://output");
        if(fileSystem.exists(path)){
            fileSystem.delete(path,true);
        }
        FileInputFormat.addInputPath(job,new Path("E:\\forTestData\\phoneFlow\\data\\forPhoneFlow"));
        FileOutputFormat.setOutputPath(job,path);
        job.setNumReduceTasks(1);
        job.waitForCompletion(true);

    }
}