NLineInputFormat案例分析与实现

最新推荐文章于 2021-06-17 08:06:26 发布

葳玟桦茗

最新推荐文章于 2021-06-17 08:06:26 发布

阅读量74

点赞数

分类专栏：大数据文章标签：大数据 hadoop

本文链接：https://blog.csdn.net/weixin_43982143/article/details/115561405

版权

大数据专栏收录该内容

12 篇文章 0 订阅

订阅专栏

理解NLineInputFormat

用于读hdfs中的文本文件，每次入读固定行数

键是文件中行的字节偏移量，值是行本身。N 是每个 Mapper 收到的输入行数

这里的键和值与TextInputFormat生成的一样

编写Mapper类

package com.dev1.nline;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class NLineMapper extends Mapper<LongWritable, Text,Text,LongWritable> {
    private LongWritable v=new LongWritable();
    private Text k=new Text();
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] words = value.toString().split(" ");
        for(String word:words) {
            k.set(word);
            v.set(1);
            context.write(k, v);
        }
    }
}

编写NLineReducer类

package com.dev1.keyvalue.nline;

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class NLineReducer extends Reducer<Text, LongWritable,Text,LongWritable> {
    private LongWritable v=new LongWritable();
    @Override
    protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
        int sum=0;
        for(LongWritable value:values){
            sum+=value.get();
        }
        v.set(sum);
        context.write(key,v);
    }
}

编写NLineDriver类

package com.dev1.nline;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class NLineDriver {
    public static void main(String[] args) throws Exception{
        Configuration configuration = new Configuration();
        Job job = Job.getInstance(configuration);
        // 7设置每个切片InputSplit中划分三条记录
        NLineInputFormat.setNumLinesPerSplit(job, 2);
        // 8使用NLineInputFormat处理记录数
        job.setInputFormatClass(NLineInputFormat.class);
        // 2设置jar包位置，关联mapper和reducer
        job.setJarByClass(NLineDriver.class);
        job.setMapperClass(NLineMapper.class);
        job.setReducerClass(NLineReducer.class);
        // 3设置map输出kv类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(LongWritable.class);
        // 4设置最终输出kv类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);
        // 5设置输入输出数据路径
        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));
        // 6提交job
        job.waitForCompletion(true);
    }
}