本站数据集:https://download.csdn.net/download/FRESHET/12649664
1.MAPPER类
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class AvgMapper extends Mapper<Object, Text, Text, IntWritable>{
@Override
protected void map(Object key, Text value, Mapper<Object, Text, Text, IntWritable>.Context context)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
// super.map(key, value, context);
String line=value.toString();
StringTokenizer stringTokenizer=new StringTokenizer(line);
while(stringTokenizer.hasMoreTokens()) {
String name=stringTokenizer.nextToken();
int score=Integer.valueOf(stringTokenizer.nextToken());
context.write(new Text(name), new IntWritable(score));
}
}
}
2.REDUCE类
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
public class AvgReduce extends Reducer<Text, IntWritable, Text, FloatWritable>{
@Override
protected void reduce(Text arg0, Iterable<IntWritable> arg1,
Reducer<Text, IntWritable, Text, FloatWritable>.Context arg2) throws IOException, InterruptedException {
// TODO Auto-generated method stub
// super.reduce(arg0, arg1, arg2);
int cnt=0;
int sum=0;
float avg=0;
for(IntWritable score:arg1) {
sum+=score.get();
cnt++;
}
avg=sum/cnt;
arg2.write(arg0, new FloatWritable(avg));
}
}
3.主类稍微麻烦点
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import com.lu.map.AvgMapper;
import com.lu.red.AvgReduce;
public class Avg {
public static void main(String[] args) throws Exception{
Configuration conf = new Configuration();
Job job = Job.getInstance(conf,"avg count");
job.setJarByClass(Avg.class);
job.setMapperClass(AvgMapper.class); // 为job设置Mapper类
job.setReducerClass(AvgReduce.class); // 为job设置Reducer类
//这回必须加了,Mapper<Object, Text, Text, IntWritable>,Reducer<Text, IntWritable, Text, FloatWritable>
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(FloatWritable.class); //这里用了FLOAT
Path in1= new Path("hdfs://192.168.1.27:9000/test/wordcnt/in/score1.txt");
Path in2= new Path("hdfs://192.168.1.27:9000/test/wordcnt/in/score2.txt");
Path in3= new Path("hdfs://192.168.1.27:9000/test/wordcnt/in/score3.txt");
Path in4= new Path("hdfs://192.168.1.27:9000/test/wordcnt/in/score4.txt");
Path in5= new Path("hdfs://192.168.1.27:9000/test/wordcnt/in/score5.txt");
Path out= new Path("hdfs://192.168.1.27:9000/test/wordcnt/out/4");
FileInputFormat.addInputPath(job, in1);
FileInputFormat.addInputPath(job, in2);
FileInputFormat.addInputPath(job, in3);
FileInputFormat.addInputPath(job, in4);
FileInputFormat.addInputPath(job, in5);
FileOutputFormat.setOutputPath(job, out);
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
结果汇总后的百分比