MapReduce经典案例Wordcount

最新推荐文章于 2024-04-04 03:18:08 发布

谷雨佳节

最新推荐文章于 2024-04-04 03:18:08 发布

阅读量226

点赞数 1

分类专栏： MapReduce 文章标签： Wordcount

本文链接：https://blog.csdn.net/guyujiajie/article/details/96176313

版权

MapReduce 专栏收录该内容

1 篇文章 0 订阅

订阅专栏

Map 阶段：

package com.hadoop;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class WordcountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{

    //private Logger logger = Logger.getLogger(WordcountMapper.class);

    Text k = new Text();

    IntWritable v = new IntWritable(1);

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        //V值转换成字符串
        String line=value.toString();

        //按照逗号切分
        String[] splits = line.split(",");

        //遍历赋值给K
        for (String word : splits) {

            k.set(word);

            context.write(k, v);
        }
    }
}

Reduce 阶段：

package com.hadoop;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class WordcountReduce extends Reducer<Text, IntWritable, Text, IntWritable>{

    int sum;

    IntWritable v = new IntWritable();

    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {

        sum = 0;

        //将单个key的value求和
        for (IntWritable count:values){

            sum +=count.get();

        }

        //赋值
        v.set(sum);

        //输出
        context.write(key,v);
    }
}

Driver 设置：

package com.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class WordcountDriver {

    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        /*
        1.获得配置信息
        2.获得任务实例
        3.获得任务Jar包类
        4.设置Map和Reduce类
        5.设置Map输出格式
        6.设置最终输出格式
        7.设置输入输出路径
        8.提交：boolean result = job.waitForCompletion(true);
        9.退出System.exit(result ? 0:1);
         */


        //获得配置并建立任务实例

        Configuration conf  = new Configuration();
        Job job = Job.getInstance(conf);

        //设置Jar加载路径
        job.setJarByClass(WordcountDriver.class);

        //设置map和reduce类
        job.setMapperClass(WordcountMapper.class);
        job.setReducerClass(WordcountReduce.class);

        //设置map输出
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        //设置最终输出类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        //设置输入输出路径(输入输出FileInputFormat,FileOutputFormat)

        FileInputFormat.setInputPaths(job,new Path("c:/input/ll.txt"));
        FileOutputFormat.setOutputPath(job,new Path(("c:/output/output")));

        //提交
        boolean result = job.waitForCompletion(true);

        System.exit(result ? 0:1);
    }
}