hadoop中MapReduce的简单操作

package com.zyc.hadoop.mapreduce.flow;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
* Created with IntelliJ IDEA.
*
* @Author: zyc2913@163.com
* @Date: 2020/9/11 14:29
* @Version: 1.0
* @Description: MapReduce实现日志查找ip、上传流量、下载流量和总流量
*/
public class WebLogLoadFlow {
    public static void main(String[] args) throws Exception {
        //1.获取job实例
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        //2.设置job运行的主类
        job.setJarByClass(WebLogLoadFlow.class);

        //3.设置Mapper的主类
        job.setMapperClass(WLLFMapper.class);

        //4.设置Reducer的主类
        job.setReducerClass(WLLFReducer.class);

        //5.设置Mapper输出的类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(WebLog.class);

        //6.设置Reducer输出的类型
        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(WebLog.class);

        //7.设置job的输入路径
        FileInputFormat.setInputPaths(job,new Path("C:\\Users\\Administrator\\Desktop\\book\\mylog"));

        //8.设置job的输出路径
        FileOutputFormat.setOutputPath(job,new Path("C:\\Users\\Administrator\\Desktop\\loadflows"));

        //9.设置MapReduce的输入格式(默认使用的是TextInPutFormat)
        job.setInputFormatClass(SequenceFileInputFormat.class);

        System.exit(job.waitForCompletion(true) ? 0 : 1 );
    }
    public static class WLLFMapper extends Mapper<Text,Text,Text,WebLog>{
        Text k = new Text();

        @Override
        protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
            for (String line: value.toString().split("\n")) {
                String[] strs = line.split(" ");
                String ip = strs[0];
                long up = 0;
                long down = 0;
                try {
                    up = Long.parseLong(strs[9]);
                    down = Long.parseLong(strs[strs.length-1]);
                }catch (Exception e){
                }
                WebLog webLog = new WebLog(ip, up, down);
                k.set(ip);
                context.write(k,webLog);
            }
        }
    }
    public static class WLLFReducer extends Reducer<Text,WebLog, NullWritable,WebLog>{
        @Override
        protected void reduce(Text key, Iterable<WebLog> values, Context context) throws IOException, InterruptedException {
            long totalUp = 0;
            long totalDown = 0;
            for (WebLog value:values) {
                totalUp += value.getUp();
                totalDown += value.getDown();
            }
            WebLog v = new WebLog(key.toString(), totalUp, totalDown);
            context.write(NullWritable.get(),v);
        }
    }
}