统计手机流量

原始数据

1363157985066   13726230503     00-FD-07-A4-72-B8:CMCC  120.196.100.82  i02.c.aliimg.com                24      27      2481    24681   200
1363157995052   13826544101     5C-0E-8B-C7-F1-E0:CMCC  120.197.40.4                    4       0       264     0       200
1363157991076   13926435656     20-10-7A-28-CC-0A:CMCC  120.196.100.99                  2       4       132     1512    200
1363154400022   13926251106     5C-0E-8B-8B-B1-50:CMCC  120.197.40.4                    4       0       240     0       200
1363157993044   18211575961     94-71-AC-CD-E6-18:CMCC-EASY     120.196.100.99  iface.qiyi.com  视频网站        15      12      1527    2106    200
1363157995074   84138413        5C-0E-8B-8C-E8-20:7DaysInn      120.197.40.4    122.72.52.12            20      16      4116    1432    200
1363157993055   13560439658     C4-17-FE-BA-DE-D9:CMCC  120.196.100.99                  18      15      1116    954     200
1363157995033   15920133257     5C-0E-8B-C7-BA-20:CMCC  120.197.40.4    sug.so.360.cn   信息安全        20      20      3156    2936    200
1363157983019   13719199419     68-A1-B7-03-07-B1:CMCC-EASY     120.196.100.82                  4       0       240     0       200
1363157984041   13660577991     5C-0E-8B-92-5C-20:CMCC-EASY     120.197.40.4    s19.cnzz.com    站点统计        24      9       6960    690     200
1363157973098   15013685858     5C-0E-8B-C7-F7-90:CMCC  120.197.40.4    rank.ie.sogou.com       搜索引擎        28      27      3659    3538    200
1363157986029   15989002119     E8-99-C4-4E-93-E0:CMCC-EASY     120.196.100.99  www.umeng.com   站点统计        3       3       1938    180     200
1363157992093   13560439658     C4-17-FE-BA-DE-D9:CMCC  120.196.100.99                  15      9       918     4938    200
1363157986041   13480253104     5C-0E-8B-C7-FC-80:CMCC-EASY     120.197.40.4                    3       3       180     180     200
1363157984040   13602846565     5C-0E-8B-8B-B6-00:CMCC  120.197.40.4    2052.flash2-http.qq.com 综合门户        15      12      1938    2910    200
1363157995093   13922314466     00-FD-07-A2-EC-BA:CMCC  120.196.100.82  img.qfc.cn              12      12      3008    3720    200
1363157982040   13502468823     5C-0A-5B-6A-0B-D4:CMCC-EASY     120.196.100.99  y0.ifengimg.com 综合门户        57      102     7335    110349  200
1363157986072   18320173382     84-25-DB-4F-10-1A:CMCC-EASY     120.196.100.99  input.shouji.sogou.com  搜索引擎        21      18      9531    2412    200
1363157990043   13925057413     00-1F-64-E1-E6-9A:CMCC  120.196.100.55  t3.baidu.com    搜索引擎        69      63      11058   48243   200
1363157988072   13760778710     00-FD-07-A4-7B-08:CMCC  120.196.100.82                  2       2       120     120     200
1363157985066   13726238888     00-FD-07-A4-72-B8:CMCC  120.196.100.82  i02.c.aliimg.com                24      27      2481    24681   200
1363157993055   13560436666     C4-17-FE-BA-DE-D9:CMCC  120.196.100.99                  18      15      1116    954     200

需求

统计出每个手机号的流量

编码实现

Mapper

public class TrafficCountMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
                String line = value.toString();
                String[] fileds = line.split("\t");
                if(fileds.length!=11) return;

                context.write(new Text(fileds[1]), new IntWritable(Integer.parseInt(fileds[9])));

        }
}

Reduce

public class TrafficCountReducer  extends Reducer< Text, IntWritable,  Text, IntWritable>{
        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
                int traffic = 0;
                for (IntWritable value : values) {
                        traffic += value.get();
                }
                context.write(key, new IntWritable(traffic));
        }
}

TrafficCount

public class TrafficCount {
        public static void main(String[] args) throws Exception{
                Configuration config = new Configuration();
                Job job = Job.getInstance(config);

                job.setJarByClass(TrafficCount.class);

                job.setMapperClass(TrafficCountMapper.class);
                job.setReducerClass(TrafficCountReducer.class);

                job.setMapOutputKeyClass(Text.class);
                job.setMapOutputValueClass(IntWritable.class);

                job.setOutputKeyClass(Text.class);
                job.setOutputValueClass(IntWritable.class);

                /*FileInputFormat.setInputPaths(job, new Path("hdfs://192.168.56.160:9000/traffic/input/traffic.dat"));
                FileOutputFormat.setOutputPath(job, new Path("hdfs://192.168.56.160:9000/traffic/output"));*/
                FileInputFormat.setInputPaths(job, new Path("/home/hasee/bigdata/traffic/input/traffic.dat"));
                FileOutputFormat.setOutputPath(job, new Path("/home/hasee/bigdata/traffic/output"));
                boolean res = job.waitForCompletion(true);
                System.exit(res?0:1);
        }
}