MapReduce作业1

作业1

绘制入门案例wordcount的图解

作业2：

wordcount的代码编写及其自定义分区器

• MyMapper

package com.qf.mr.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
   

    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
   
        //map方法的key就是k1, 行偏移量不需要，因此不需要处理。只需要处理value，因为value就是v1，行记录
        //一对k1,v1就会调用一次map函数，因此map方法执行的次数和行记录数有关系
        //1: 将value的类型转为java的String类型
        String line = value.toString();
        //2: 使用空格对行记录进行切分成字符串数组
        String[] words = line.split(" ");
        //3: 遍历数组
        for (String word : words) {
   
            //要将word类型转为Text类型 ，当成k2      IntWritable类型的1作为value
            Text k2 = new Text(word);
            IntWritable v2 = new IntWritable(1);
            //4 将k2,v2,作为输出数据写出去，写到shuffle流程中的缓存区
            context.write(k2,v2);
        }
    }
}

• MyReducer

package com.qf.mr.wordcount;

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class MyReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
   
    @Override
    protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
   
        int sum = 0;
        for (IntWritable value : values) {
   
            //累加
            sum+=value.get();
        }
        //累加的结果sum就是v3, 转成IntWritable
        IntWritable v3 = new IntWritable(sum);
        //k2,就是k3,写出去
        context.write(key,v3);
    }
}

• MyDrive

package com.qf.mr.wordcount;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io