public class counts {
static {
System.setProperty("hadoop.home.dir","D:\\hadoop\\hadoop-2.9.2");
}
public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "wuyithreeone");
//job.setJarByClass(counts.class);
//获取输入
FileInputFormat.addInputPaths(job,args[0]);
//mapper计算
job.setMapperClass(MyMapper.class);
//shuffle计算
//reduce计算
job.setReducerClass(MyReduce.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
//获取输出
FileOutputFormat.setOutputPath(job,new Path(args[1]));
//提交
boolean b = job.waitForCompletion(true);
System.out.println(b?1:0);
}
public static class MyMapper extends Mapper<LongWritable,Text,Text,Text>{
@Override
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
String s = value.toString();
String fileName = ((FileSplit) context.getInputSplit()).getPath().getName();
String[] split = s.split(" ");
if ("a.txt".equals(fileName)){
context.write(new Text(split[0]),new Text("a.txt"));
context.write(new Text(split[1]),new Text("a.txt"));
}else if ("b.txt".equals(fileName)){
context.write(new Text(split[0]),new Text("b.txt"));
context.write(new Text(split[1]),new Text("b.txt"));
}else if ("c.txt".equals(fileName)) {
context.write(new Text(split[0]), new Text("c.txt"));
context.write(new Text(split[1]), new Text("c.txt"));
}
}
}
public static class MyReduce extends Reducer<Text,Text,Text,Text>{
@Override
protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {
long a = 0L;
long b = 0L;
long c = 0L;
for (Text value:values){
String s = value.toString();
if ("a.txt".equals(s)){
a+=1;
}else if("b.txt".equals(s)){
b+=1;
}else if("c.txt".equals(s)){
c+=1;
}
}
context.write(new Text(key),new Text("a.txt->"+a+",b.txt->"+b+",c.txt->"+c));
}
}
}
mapreduce统计多文件的单词个数
最新推荐文章于 2020-12-20 17:05:56 发布