实验三:要求请按照部门号,统计不同部门员工工资的总额,使用MapReduce来实现该实验。实验数据如下:
在eclipce里面编辑好Hadoop代码打成jar到本地,然后上传到hdfs上,建议上传到[root@WFB ~]# cd /training/hadoop-2.7.3/etc/hadoop/目录下
package com.liusong.MR;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class SalaryTotalMain {
public static void main(String[] args) throws Exception {
//1、创建一个任务job =map + reduce
Job job = Job.getInstance(new Configuration());
//2、指定任务的入口
job.setJarByClass(SalaryTotalMain.class);
//3、指定任务的map和输出的数据类型
job.setMapperClass(SalaryTotalMapper.class);
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(LongWritable.class);
//4、指定任务的reduce和输出的数据类型
job.setReducerClass(SalaryTotalReducer.class);
job.setOutputKeyClass(LongWritable.class);
job.setOutputValueClass(LongWritable.class);
//5、指定输入和输出的HDFS路径
FileInputFormat.setInputPaths(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
//6、提交任务
job.waitForCompletion(true);
}
}
package com.liusong.MR;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
public class SalaryTotalMapper extends Mapper<LongWritable, Text, LongWritable, LongWritable>{
@Override
protected void map(LongWritable k1,Text v1, Context context) throws IOException,InterruptedException{
//数据:7654,MARTIN,SALESMAN,7698,1981/9/28,1250/1400/30
String data = v1.toString();
//分词
String[] words = data.split(",");
//输出:k2 部门号,v2:员工薪水
context.write(new LongWritable(Long.parseLong(words[7])), new LongWritable(Long.parseLong(words[5])));
}
}
package com.liusong.MR;
import java.io.IOException;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.mapreduce.Reducer;
public class SalaryTotalReducer extends Reducer<LongWritable, LongWritable, LongWritable, LongWritable> {
@Override
protected void reduce(LongWritable k3,Iterable<LongWritable> v3,Context context) throws IOException,InterruptedException{
//得到v3,代表一个部门中一个员工的薪水
long total = 0;
for (LongWritable v : v3) {
total = total + v.get();
}
//输出k4部门v4总额
context.write(k3, new LongWritable(total));
}
}
上传后吧要上传的工作表上传到web上面去