现有手机流量日志,记录有手机号码、MAC地址、IP地址、浏览内容、上行流量、下行流量等,
A.分析每个手机号的上行流量、下行流量和总流量(需要一个流量实体)
B.将结果按总流量排序,总流量相同按下行流量排序
C.求流量排行前三的手机号
一、Flow实体
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
/*
自定义数据类型时一定要注意,要是想在Map或Reduce阶段输出
①做值必须实现Writable接口
②做键必须实现WritableComparable接口
③正确重写序列化方法
④正确重写比较方法
*/
public class Flow implements WritableComparable<Flow>{
private int upFlow;
private int downFlow;
private int sumFlow;
private String phoneNumber="";//必须赋初值,防止序列化失败
public Flow(){
}
public Flow (Flow f){
this.phoneNumber=f.phoneNumber;
this.upFlow=f.upFlow;
this.downFlow=f.downFlow;
this.sumFlow=f.sumFlow;
}
public Integer getUpFlow() {
return upFlow;
}
public void setUpFlow(Integer upFlow) {
this.upFlow = upFlow;
}
public Integer getDownFlow() {
return downFlow;
}
public void setDownFlow(Integer downFlow) {
this.downFlow = downFlow;
}
public Integer getSumFlow() {
this.sumFlow=this.upFlow+this.downFlow;
return sumFlow;
}
public void setSumFlow(Integer sumFlow) {
this.sumFlow = sumFlow;
}
public String getPhoneNumber() {
return phoneNumber;
}
public void setPhoneNumber(String phoneNumber) {
this.phoneNumber = phoneNumber;
}
@Override
public String toString() {
return phoneNumber+"\t"+upFlow+"\t"+downFlow+"\t"+sumFlow;
}
public int compareTo(Flow o) {
if(this.sumFlow==o.sumFlow){
return o.downFlow-this.downFlow;
}else {
return o.sumFlow-this.sumFlow;
}
}
/*
重写序列化方法时读写顺序要一致!!!类型要一致!!!
*/
public void write(DataOutput dataOutput) throws IOException {
dataOutput.writeInt(upFlow);
dataOutput.writeInt(downFlow);
dataOutput.writeInt(sumFlow);
dataOutput.writeUTF(phoneNumber);
}
public void readFields(DataInput dataInput) throws IOException {
upFlow=dataInput.readInt();
downFlow=dataInput.readInt();
sumFlow=dataInput.readInt();
phoneNumber=dataInput.readUTF();
}
}
二、分析每个手机的上行流量、下行流量和总流量
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
/*
分析每个手机的上行流量、下行流量和总流量
需要自己制作一个流量实体
*/
public class ForPhoneFlow {
public static class ForMapper extends Mapper<LongWritable,Text,Text,Flow> {
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line=value.toString();
Flow flow=new Flow();
String []strs=line.split("\t");
flow.setPhoneNumber(strs[1]);
flow.setUpFlow(Integer.parseInt(strs[8]));
flow.setDownFlow(Integer.parseInt(strs[9]));
flow.getSumFlow();
context.write(new Text(strs[1]),flow);
}
}
public static class ForReducer extends Reducer<Text,Flow,Flow,NullWritable> {
@Override
protected void reduce(Text key, Iterable<Flow> values, Context context) throws IOException, InterruptedException {
int sumUp=0;
int sumDown=0;
for(Flow f:values){
sumUp+=f.getUpFlow();
sumDown+=f.getDownFlow();
}
Flow flow=new Flow();
flow.setPhoneNumber(key.toString());
flow.setUpFlow(sumUp);
flow.setDownFlow(sumDown);
flow.getSumFlow();
context.write(flow,NullWritable.get());
}
}
public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException, URISyntaxException {
Job job= Job.getInstance();
job.setMapperClass(ForMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Flow.class);
job.setReducerClass(ForReducer.class);
job.setOutputKeyClass(Flow.class);
job.setOutputValueClass(NullWritable.class);
FileInputFormat.addInputPath(job,new Path("E://forTestData//phoneFlow//data"));
FileSystem fileSystem=FileSystem.get(new URI("file:e//output"),new Configuration());
Path path=new Path("e://output");
if(fileSystem.exists(path)){
fileSystem.delete(path,true);
}
FileOutputFormat.setOutputPath(job,path);
job.setNumReduceTasks(1);//若要求出全局TopN,map处理完的数据只能交付给一个reduce
job.waitForCompletion(true);
}
}
/*
中间还很出现了两个小错误:
①Unable to initialize any output collector
注意一下通过Job给Map或Reduce设置的输出类型和写类的时候的那个泛型是否一致!
因为我中间改过输出类型,只改了上面没改下面
②Unable to initialize any output collector
是因为输出类型是自定义的,但是我没有实现WritableComparable接口
*/
三、按用户的总流量排序,总流量相同按下行流量排序
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
/*
分析每个手机的上行流量、下行流量和总流量
并按照总流量排序
如果总流量相同,按下行流量排序
*/
public class ForSortPhoneFlow {
public static class ForMapper extends Mapper<LongWritable,Text,Text,Flow>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line=value.toString();
String []strs=line.split("\t");
Flow flow=new Flow();
flow.setPhoneNumber(strs[1]);
flow.setUpFlow(Integer.parseInt(strs[8]));
flow.setDownFlow(Integer.parseInt(strs[9]));
flow.getSumFlow();
context.write(new Text(strs[1]),flow);
}
}
public static class ForReducer extends Reducer<Text,Flow,Flow,NullWritable>{
List<Flow> list=new ArrayList<Flow>();
@Override
protected void reduce(Text key, Iterable<Flow> values, Context context) throws IOException, InterruptedException {
int sumUp = 0;
int sumDown = 0;
for (Flow f : values) {
sumUp += f.getUpFlow();
sumDown += f.getDownFlow();
}
Flow f = new Flow();
f.setPhoneNumber(key.toString());
f.setUpFlow(sumUp);
f.setDownFlow(sumDown);
f.getSumFlow();
list.add(f);
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
Collections.sort(list, new Comparator<Flow>() {
public int compare(Flow o1, Flow o2) {
if(o1.getSumFlow()==o2.getSumFlow()){
return o2.getDownFlow()-o1.getDownFlow();
}else {
return o2.getSumFlow()-o1.getSumFlow();
}
}
});
for(Flow f:list){
context.write(f, NullWritable.get());
}
}
}
public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException {
Job job= Job.getInstance();
job.setMapperClass(ForMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Flow.class);
job.setReducerClass(ForReducer.class);
job.setOutputKeyClass(Flow.class);
job.setOutputValueClass(NullWritable.class);
FileSystem fileSystem=FileSystem.get(new URI("file:E://output"),new Configuration());
Path path=new Path("E://output");
if(fileSystem.exists(path)){
fileSystem.delete(path,true);
}
FileInputFormat.addInputPath(job,new Path("E://forTestData//phoneFlow//data"));
FileOutputFormat.setOutputPath(job,path);
job.setNumReduceTasks(1);
job.waitForCompletion(true);
}
}
四、在(二)的输出结果的基础上,计算手机流量排行前三的手机号
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
/*
在ForPhoneFlow的基础上
计算手机流量排行前三的手机号
*/
public class ForTopThreePhone {
public static class ForMapper extends Mapper<LongWritable,Text,Text,Flow>{
List<Flow> list=new ArrayList<Flow>();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String line=value.toString();
String []strs=line.split("\t");
Flow flow=new Flow();
flow.setPhoneNumber(strs[0]);
flow.setUpFlow(Integer.parseInt(strs[1]));
flow.setDownFlow(Integer.parseInt(strs[2]));
flow.getSumFlow();
list.add(flow);
if(list.size()==4){
Collections.sort(list);
list.remove(3);
}
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
for(int i=0;i<3;i++){
context.write(new Text(list.get(i).getPhoneNumber()),list.get(i));
}
}
}
public static class ForReducer extends Reducer<Text,Flow,Flow,NullWritable>{
List<Flow> list=new ArrayList<Flow>();
@Override
protected void reduce(Text key, Iterable<Flow> values, Context context) throws IOException, InterruptedException {
/*
往List集合里面添加的时候,一定要新建一个对象
不然加到List中的都是同一个引用,List里的存的对象都是同一个对象
*/
list.add(new Flow(values.iterator().next()));
}
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
Collections.sort(list);
for(int i=0;i<3;i++){
context.write(list.get(i),NullWritable.get());
}
}
}
public static void main(String[] args) throws IOException, URISyntaxException, ClassNotFoundException, InterruptedException {
Job job=Job.getInstance();
job.setMapperClass(ForMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(Flow.class);
job.setReducerClass(ForReducer.class);
job.setOutputKeyClass(Flow.class);
job.setOutputValueClass(NullWritable.class);
FileSystem fileSystem=FileSystem.get(new URI("file:E://output"),new Configuration());
Path path=new Path("E://output");
if(fileSystem.exists(path)){
fileSystem.delete(path,true);
}
FileInputFormat.addInputPath(job,new Path("E:\\forTestData\\phoneFlow\\data\\forPhoneFlow"));
FileOutputFormat.setOutputPath(job,path);
job.setNumReduceTasks(1);
job.waitForCompletion(true);
}
}