【Hadoop--手机流量分析】

实习日记2024.6.5

手机流量分析

搭建好环境后，开发步骤如下

一、在虚拟机中装上idea并启动

[root@my2308-host idea-IC-211.7628.21]# cd bin
[root@my2308-host bin]# ./idea.sh

---

二、配置maven并导入相关依赖

在pom.xml中导入依赖

 <dependencies>
        <!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.2.0</version>
        </dependency>
    </dependencies>

三、创建Java文件

创建四个java类

创建两个分别叫做liuliang2和liuliang2_2的文件，且分别创建四个类和五个类
.FlowBean.java

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.Writable;

// bean对象要实例化
public class FlowBean implements Writable {

    private long upFlow;
    private long downFlow;
    private long sumFlow;

    // 反序列化时，需要反射调用空参构造函数，所以必须有
    public FlowBean() {
        super();
    }

    public FlowBean(long upFlow, long downFlow) {
        super();
        this.upFlow = upFlow;
        this.downFlow = downFlow;
        this.sumFlow = upFlow + downFlow;
    }

    public long getSumFlow() {
        return sumFlow;
    }

    public void setSumFlow(long sumFlow) {
        this.sumFlow = sumFlow;
    }

    public long getUpFlow() {
        return upFlow;
    }

    public void setUpFlow(long upFlow) {
        this.upFlow = upFlow;
    }

    public long getDownFlow() {
        return downFlow;
    }

    public void setDownFlow(long downFlow) {
        this.downFlow = downFlow;
    }

    /**
     * 序列化方法
     *
     * @param out
     * @throws IOException
     */
    @Override
    public void write(DataOutput out) throws IOException {
        out.writeLong(upFlow);
        out.writeLong(downFlow);
        out.writeLong(sumFlow);
    }

    /**
     * 反序列化方法
     注意反序列化的顺序和序列化的顺序完全一致
     *
     * @param in
     * @throws IOException
     */
    @Override
    public void readFields(DataInput in) throws IOException {
        upFlow = in.readLong();
        downFlow = in.readLong();
        sumFlow = in.readLong();
    }

    @Override
    public String toString() {
        return upFlow + "\t" + downFlow + "\t" + sumFlow;
    }

    public void set(long upFlow, long downFlow) {
        this.upFlow = upFlow;
        this.downFlow = downFlow;
        this.sumFlow = upFlow + downFlow;
    }
}

FlowDriver.java

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;


public class FlowDriver {

    public static void main(String[] args) throws Exception {

        //1.获取配置信息
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        //2.获取jar包信息
        job.setJarByClass(FlowDriver.class);

        //3.配置mapper、reducer类
        job.setMapperClass(FlowMapper.class);
        job.setReducerClass(FlowReducer.class);

        //4.配置mapper输出key、value值
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(FlowBean.class);

        //5.配置输出key、value值
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(FlowBean.class);

        //设置分区
        job.setPartitionerClass(ProvincePartitioner.class);

        //设置Reducenum,依据是看flowpartitioner里分了几个区
        job.setNumReduceTasks(4);

        //6.配置输入路径和输出路径
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        //7.提交
        boolean result = job.waitForCompletion(true);
        System.exit(result?0:1);
    }
}

FlowMapper.java

import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

public class FlowMapper extends Mapper<LongWritable, Text, Text, FlowBean>{

    //不能在map方法中new对象，map方法执行频率高，内存消耗大。这也就是需要在bean对象中要有一个空构造方法的原因
    FlowBean bean = new FlowBean();
    Text k = new Text();

    @Override
    protected void map(LongWritable key, Text value, Context context)
            throws IOException, InterruptedException {

        //1.获取一行数据（）
        String line = value.toString();

        //2.截取字段（1、2可归结为一大步：对输入数据的处理）
        String[] fields = line.split("\t");

        //3.封装bean对象，获取电话号码（第二大步：具体的业务逻辑）
        String phoneNum = fields[0];
        long upFlow = Long.parseLong(fields[fields.length - 3]);
        long downFlow = Long.parseLong(fields[fields.length - 2]);
        //在map方法中new对象是不好的，因为在输入数据时，每读一行数据，会执行以下map方法，这一造成内存消耗很大
        //FlowBean bean = new FlowBean(upFlow,downFlow);
        bean.set(upFlow, downFlow);
        k.set(phoneNum);

        //4.写出去（第三大步：将数据输出出去，key和value分别是什么，规定清楚）
        //context.write(new Text(phoneNum), bean);
        context.write(k, bean);
    }
}

FlowReducer.java

import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

public class FlowReducer extends Reducer<Text, FlowBean, Text, FlowBean>{

    @Override
    protected void reduce(Text key, Iterable<FlowBean> values, Context context)
            throws IOException, InterruptedException {
        //1.计算总的流量
        long sum_upFlow = 0;
        long sum_downFlow = 0;
        for(FlowBean bean : values){
            sum_upFlow += bean.getUpFlow();
            sum_downFlow += bean.getDownFlow();
        }

        //2.输出
        context.write(key, new FlowBean(sum_upFlow,sum_downFlow));
    }
}

ProvincePartitioner

import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Partitioner;

public class ProvincePartitioner extends Partitioner<Text, FlowBean> {
    @Override
    public int getPartition(Text key, FlowBean value, int numPartitions) {
        // 1 获取电话号码的前三位
        String preNum = key.toString().substring(0, 3);

        int partition;

        // 2 判断是哪个前缀
        if (preNum.startsWith("13")) {
            partition = 0; // 13* 前缀
        } else if (preNum.startsWith("15")) {
            partition = 1; // 15* 前缀
        } else {
            partition = 2; // 其他前缀
        }

        return partition;
    }
}

四、启动集群和节点

启动集群命令为：

./start-all.sh

查看节点命令为：

jps

如图所示则启动成功

五、将两个Java文件打包上传到虚拟机的Linux操作系统里面

打包步骤：File --> Project Structure --> Artifacts --> + -->JAR --> From modules with dependencies
随后在Main Class中选择 FlowDriver

创建文件夹（在Linux虚拟机终端或者xshell里都可以）

[root@my2308-host sbin]# hadoop fs -mkdir -p /zs/java/input/input8

将流量日志文件上传到创建的文件里

[root@my2308-host ~]# hadoop fs -put data/access.log /zs/java/input/input8/

六、在集群中查看执行结果

输出：

完成