WCDRIVER
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.io.IOException;
public class WCDriver {
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
//创建JOB对象
Job job = Job.getInstance(new Configuration());
//设置参数
//设置jar加载类型
job.setJarByClass(WCDriver.class);
//设置mapper和reducer类
job.setMapperClass(WCMapper.class);
job.setReducerClass(WCReducer.class);
//设置MAPPER输出的key,value的类型
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//设置最终输出的KEY,value的类型(在这里指的是reduce输出的KEY,value)
job.setOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
//设置读取的数据的路径--如果指定的目录那马会将整个目录中所有的文件进行运算
FileInputFormat.setInputPaths(job,new Path("D:\\lianxizhuanyong"));
//设置输出的路劲---输出的路径必须不存在
FileOutputFormat.setOutputPath(job,new Path("D:\\lianxizhuanyong3"));
//提交参数
job.waitForCompletion(true);
}
}
WCREDUCER
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
import java.io.IOException;
public class WCReducer extends Reducer<Text, IntWritable, Text,IntWritable> {
//封装的value
private IntWritable outvalue= new IntWritable();
@Override
protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
int sum=0;
// 1.遍历所有的value
for(IntWritable value:values){
//2.对所有valu进行累加
int v = value.get();//将Intwritable转换成基本数据类型
sum +=v;
}
//封装key value
outvalue.set(sum);//将基本数据类型转换成Intweitrable(实际上是赋值)
//写出去
context.write(key,outvalue);
}
}
WCMAPPE
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
public class WCMapper extends Mapper<LongWritable, Text,Text, IntWritable> {
private Text outkey = new Text();
private IntWritable outvalue = new IntWritable();
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//将value转换成字符串类型
String line=value.toString();
//对字符串进行切割
String[] words = line.split(" ");
for(String word:words){
//封装KV
outkey.set(word);
outvalue.set(1);
//写出去
context.write(outkey,outvalue);
}
}
}