IpUv的使用

数据：

1.1.1.1 a
2.2.2.2 a
1.1.1.1 b
1.1.1.1 c
2.2.2.2 f
3.3.3.3 gg

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;


public class IpUv {
    //本地运行需要的方法
    static {
        System.setProperty("hadoop.home.dir", "D:\\soft\\hadoop\\hadoop-2.9.2");
    }

    public static class MyMapper1 extends Mapper<LongWritable, Text, Text, Text> {
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//            String ip = value.toString().split(" ")[0];
            String s = value.toString();
            String[] split = s.split("\\s");
            String ip=split[0];
            context.write(new Text(ip.trim()), new Text("1"));
        }
    }

    public static class MyReducer1 extends Reducer<Text, Text, Text, Text> {
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            context.write(key, new Text("1"));
        }
    }

    public static class MyMapper2 extends Mapper<LongWritable, Text, Text, Text> {
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String ip = value.toString().split("\t")[0];
            context.write(new Text("uv"), new Text("1"));
        }
    }

    public static class MyReducer2 extends Reducer<Text, Text, Text, Text> {
        @Override
        protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
            long sum = 0L;
            for (Text value : values) {
                sum += Long.parseLong(value.toString());
            }
            context.write(key, new Text(String.valueOf(sum)));
        }
    }

    /**
     * 驱动方法
     * @param args
     */
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        //0.初始化一个job
        Configuration conf = new Configuration();

        Job job1 = Job.getInstance(conf, "ip-uv");
        job1.setJarByClass(IpUv.class);
        //1.输入文件
        FileInputFormat.addInputPath(job1, new Path(args[0]));
        //2.map并行计算
        job1.setMapperClass(MyMapper1.class);
        //3.shuffle流程(内部实现)
        //4.reduce计算
        job1.setReducerClass(MyReducer1.class);
        job1.setOutputKeyClass(Text.class);
        job1.setOutputValueClass(Text.class);
        //5.输出文件
        FileOutputFormat.setOutputPath(job1, new Path(args[1]));
        //6.提交作业(总入口)
        boolean result = job1.waitForCompletion(true);
       if (result) {
           Job job2 = Job.getInstance(conf, "ip-uv-2");
           job2.setJarByClass(IpUv.class);
           //1.输入文件
           FileInputFormat.addInputPath(job2, new Path(args[1]));
           //2.map并行计算
           job2.setMapperClass(MyMapper2.class);
           //3.shuffle流程(内部实现)
           //4.reduce计算
           job2.setReducerClass(MyReducer2.class);
           job2.setOutputKeyClass(Text.class);
           job2.setOutputValueClass(Text.class);
           //5.输出文件
           FileOutputFormat.setOutputPath(job2, new Path(args[2]));
           //6.提交作业(总入口)
           result = job2.waitForCompletion(true);
       }
       System.exit(result ? 1 : 0);
    }
}

pv结果：

1.1.1.1	1
2.2.2.2	1
3.3.3.3	1

uv结果：

uv	3