【hadoop学习】在给定的文件中统计输出每一个字符串出现的总次数

最新推荐文章于 2021-12-20 11:35:47 发布

鹰川

最新推荐文章于 2021-12-20 11:35:47 发布

阅读量1.5k

点赞数

分类专栏：笔记文章标签： hadoop mapreduce big data

本文链接：https://blog.csdn.net/weixin_46115600/article/details/121865899

版权

笔记专栏收录该内容

5 篇文章 0 订阅

订阅专栏

开始学习hadoop，过程十分艰难，借鉴了一些资料，代码仅供自己学习参考

需要先虚拟机dfs和yarn群起，网页打开hadoop100，然后操作

以下是代码实现

Mapper:

package mapreduce1210;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.awt.image.ImageProducer;
import java.io.IOException;

public class it863_01_Mapper extends Mapper<LongWritable, Text,Text, IntWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        //todo mao调用时机  每一行数据调用一次map  每一个k，v调用一次map
        String[] s = value.toString().replace("、","，").split("，");
        for (String s1:s){
            context.write(new Text(s1),new IntWritable(1));
        }


    }
}

Reducer:

package mapreduce1210;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class it863_01_Reducer extends Reducer<Text, IntWritable,Text, LongWritable>{
    //todo reduce调用时机   每一个k对应的k，v调用一次reduce
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
        long count = 0;
        for (IntWritable s_count:values){
            s_count.get();
            count++;
        }
        context.write(key,new LongWritable(count));
    }
}

Driver:

package mapreduce1210;

import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.FileInputStream;
import java.io.IOException;

public class it863_01_Driver {
    public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
        Job job = Job.getInstance();

        //设置要执行的类的路径
        job.setJarByClass(it863_01_Driver.class);

        //设定要执行的mapper类   和k，v
        job.setMapperClass(it863_01_Mapper.class);
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);
        //设定要执行的reducer
        job.setReducerClass(it863_01_Reducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(LongWritable.class);


        //设置文件输入路径
        FileInputFormat.addInputPath(job,new Path("D:\\863863\\day1210\\西游记(1).txt"));

        //设置文件输出路径
        FileOutputFormat.setOutputPath(job,new Path("D:\\863863\\day1210\\out"));
        boolean success = job.waitForCompletion(true);
        if (success){
            System.exit(1);

        }else {
            System.exit(0);
        }
    }

}