Hadoop之 KeyValueInputFormat

数据

lisi 数学 78
lisi 语文 89
lisi 英语 89
zhangsan 数学 56
zhangsan 语文 87
zhangsan 英语 56

自定义Mapper

package com.test;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class TestMapper extends Mapper<Text, Text,Text, IntWritable> {
    @Override
    protected void map(Text key, Text value, Context context) throws IOException, InterruptedException {
        context.write(key, new IntWritable(1));
    }
}

自定义Reducer

package com.test;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class TestReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        int sum = 0;
        for (IntWritable va : values) {
           sum+=va.get();
        }
        context.write(key, new IntWritable(sum));
    }
}

自定义Driver

package com.test;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.KeyValueLineRecordReader;
import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class TestDriver {
    public static void main(String[] args) throws Exception {
        // 数据输入路径和输出路径
        args = new String[2];
        args[0] = "src/main/resources/input";
        args[1] = "src/main/resources/output";
        Configuration cfg = new Configuration();// 读取配置文件
        cfg.set("mapreduce.framework.name", "local");
        cfg.set("fs.defaultFS", "file:///");
        cfg.set(KeyValueLineRecordReader.KEY_VALUE_SEPERATOR, " ");
        final FileSystem filesystem = FileSystem.get(cfg);
        if (filesystem.exists(new Path(args[0]))) {
            filesystem.delete(new Path(args[1]), true);
        }
        // 新建一个任务
        Job job = Job.getInstance(cfg);
        job.setJarByClass(TestDriver.class);  // 设置主类
        job.setInputFormatClass(KeyValueTextInputFormat.class);//设置输入格式
        job.setMapperClass(Testmapper.class);   // Mapper
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        job.setNumReduceTasks(0);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        FileInputFormat.addInputPath(job, new Path(args[0]));        // 输入路径
        FileOutputFormat.setOutputPath(job, new Path(args[1]));        // 输出路径
        // 提交任务
        int ec = job.waitForCompletion(true) ? 0 : 1;
        System.exit(ec);
    }
}