WordCount

WordCount

处理过程：数据HDFS=>Mapper处理阶段=>Reducer处理阶段=>HDFS

WordCountMapper ：重写方法，获取数据，将获取的数据进行按一定的规则的拆分，（k1:偏移量；v1:数据；k2:具体的单词；v2:每单词记一次数）
代码如下：
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class WordCountMapper extends Mapper<LongWritable,Text,Text,IntWritable>{
@Override
protected void map(LongWritable key1, Text value1, Context context)
throws IOException, InterruptedException {
/*
*context 表示Mapper的上下文
* 上文：HDFS
* 下文：Mapper
*/
//数据：I love Beijing
String data = value1.toString();
//分词
String[] words =data.split(" ");
//输出K2，V2
for(String w:words){
context.write(new Text(w),new IntWritable(1));
}
}
}
WordCountReducer ：对出现的单词的次数进行统计求和。(k3:具体的单词；v3:单词出现一次记一次的集合；k4:具体单词；v4:对v3的一个求和)
代码如下：
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class WordCountReducer extends Reducer<Text,IntWritable,Text,IntWritable>{
@Override
protected void reduce(Text k3, Iterable v3, Context context) throws IOException, InterruptedException {
/*
*context是reduce的上下文
* 上文
* 下文
*/
//对V3求和
int total = 0;
for(IntWritable v:v3){
total +=v.get();
}
//输出 k4单词 v4频率
context.write(k3,new IntWritable(total));
}
}
WordCountMain ：1：创建一个任务job
2：指定job的mapper以及输出的类型
3：指定job的reducer以及输出的类型
4：指定job的输入以及输出
5：运行job
代码如下：
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

public class WordCountMain {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
//创建一个job
Job job = Job.getInstance(new Configuration());
job.setJarByClass(WordCountMain.class);//main方法所在的class
//指定job的mapper和输出的类型<k2,v2>
job.setMapperClass(WordCountMapper.class);
job.setMapOutputKeyClass(Text.class);//k2的类型
job.setOutputValueClass(IntWritable.class);//v2的类型
//指定job的reducer和输出的类型<k4,v4>
job.setReducerClass(WordCountReducer.class);
job.setOutputKeyClass(Text.class);//k4类型
job.setOutputValueClass(IntWritable.class);//v4类型
//指定job的输入和输出
FileInputFormat.setInputPaths(job,new Path(args[0]));
FileOutputFormat.setOutputPath(job,new Path(args[1]));
//执行job
job.waitForCompletion(true);
}
}