MyEclipse+MapReduce小作业

数据：
computer,huangxiaoming,85,86,41,75,93,42,85
computer,xuzheng,54,52,86,91,42
computer,huangbo,85,42,96,38
english,zhaobenshan,54,52,86,91,42,85,75
english,liuyifei,85,41,75,21,85,96,14
algorithm,liuyifei,75,85,62,48,54,96,15
computer,huangjiaju,85,75,86,85,85
english,liuyifei,76,95,86,74,68,74,48
english,huangdatou,48,58,67,86,15,33,85
algorithm,huanglei,76,95,86,74,68,74,48
algorithm,huangjiaju,85,75,86,85,85,74,86
computer,huangdatou,48,58,67,86,15,33,85
english,zhouqi,85,86,41,75,93,42,85,75,55,47,22
english,huangbo,85,42,96,38,55,47,22
algorithm,liutao,85,75,85,99,66
computer,huangzitao,85,86,41,75,93,42,85
math,wangbaoqiang,85,86,41,75,93,42,85
computer,liujialing,85,41,75,21,85,96,14,74,86
computer,liuyifei,75,85,62,48,54,96,15
computer,liutao,85,75,85,99,66,88,75,91
computer,huanglei,76,95,86,74,68,74,48
english,liujialing,75,85,62,48,54,96,15
math,huanglei,76,95,86,74,68,74,48
math,huangjiaju,85,75,86,85,85,74,86
math,liutao,48,58,67,86,15,33,85
english,huanglei,85,75,85,99,66,88,75,91
math,xuzheng,54,52,86,91,42,85,75
math,huangxiaoming,85,75,85,99,66,88,75,91
math,liujialing,85,86,41,75,93,42,85,75
english,huangxiaoming,85,86,41,75,93,42,85
algorithm,huangdatou,48,58,67,86,15,33,85
algorithm,huangzitao,85,86,41,75,93,42,85,75
题目：
统计每门课程参考学生的平均分，并且按课程存入不同的结果文件，要求一门课程一个结果文件，并且按平均分从高到低排序，分数保留一位小数。

package 竞赛.studentScore;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Partitioner; 
public class CoursePartitioner extends Partitioner<CourseBean, NullWritable>{ 	
	@Override
	public int getPartition(CourseBean key, NullWritable value, int numPartitions) {
		if("algorithm".equals(key.getCourse())){
			return 0;
		}else if("computer".equals(key.getCourse())){
			return 1;
		}else if("english".equals(key.getCourse())){
			return 2;
		}else{
			return 3;
		}
	} 
}

package 竞赛.studentScore;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.io.WritableComparable;
 public class CourseBean implements WritableComparable<CourseBean>{
	private String course; //课程名
	private String name; //学生姓名
	private float avg; //平均分	
	public String getCourse() {
		return course;//获得课程名
	}
	public void setCourse(String course) {
		this.course = course;//课程名赋值
	}
	public String getName() {//获得姓名
		return name;
	}
	public void setName(String name) {
		this.name = name;//姓名赋值
	}
	public float getAvg() {
		return avg;//获得平均分
		}
	public void setAvg(float avg) {
		this.avg = avg;//平均分赋值
	}	
	public CourseBean(String course, String name, float avg) {
		super();
		this.course = course;
		this.name = name;
		this.avg = avg;
	}	
	public CourseBean() {}		
	@Override//通过toString方法自定义输出类型
	public String toString() {
		return course + "\t" + name + "\t" + avg;}	
	@Override//序列号
	public void write(DataOutput out) throws IOException {
		out.writeUTF(course);
		out.writeUTF(name);
		out.writeFloat(avg);
	}	
	@Override//反序列化
	public void readFields(DataInput in) throws IOException {
		course = in.readUTF();
		name = in.readUTF();
		avg = in.readFloat();
	}	
	@Override//比较规则
	public int compareTo(CourseBean o) {
		float flag = o.avg - this.avg;
		return flag > 0.0f ? 1:-1;
	}
 }

package 竞赛.studentScore;

import java.io.IOException;
import java.text.DecimalFormat; 
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; 
/**
 * 统计每门课程的参考人数和课程平均分
 * 考虑到要需求要根据课程进行分组并对平均值进行排序，这里使用自定义bean的形式来进行处理
 * 因为要将数据根据课程进行分区并写入到不容的文件中，所以这里使用自定partitioner组件进行分区
 * 要注意的是这时候就要设置reduceTask的个数
 */
public class CourseTwo {
	static Text text = new Text(); 
	public static class MyMapper extends Mapper<LongWritable, Text, CourseBean, NullWritable>{
		@Override
		protected void map(LongWritable key, Text value,Context context)
				throws IOException, InterruptedException {
			String[] lines = value.toString().split(",");//computer,huangxiaoming,85,86,41,75,93,42,85
			long sum = 0L;
			long totalTimes = lines.length-2;//totalTimes=7
			for (int i = 2; i < lines.length; i++) {
				sum += Long.parseLong(lines[i]);//85+86+41+75+93+42+85
			}
			//格式化平均分使用，保留一位有效小数
			DecimalFormat df=new DecimalFormat(".0");
			//计算某个学生在某门课程中的平均分
			float avg = sum*1.0f/totalTimes;
			String b = df.format(avg);
			//构建mapper输出的key          课程名       姓名           平均分
			CourseBean cb = new CourseBean(lines[0],lines[1],Float.parseFloat(b));	
			context.write(cb, NullWritable.get());
		}
	}	
	public static class MyReducer extends Reducer<CourseBean, NullWritable,CourseBean, NullWritable>{
		@Override
		protected void reduce(CourseBean key, Iterable<NullWritable> values,Context context)
				throws IOException, InterruptedException {
			//因为自定义了分区组件，自定义类型有排序规则，所以这里直接输出就可以了
			for (NullWritable nullWritable : values) {
				context.write(key, nullWritable);
			}
		}
	}	
	public static void main(String[] args) throws Exception {
		String input = "hdfs://192.168.216.128:9000/home/master/hadoop/input/studentsScore.txt";
	    String output = "hdfs://192.168.216.128:9000/home/master/hadoop/output2/"; 
		Configuration conf = new Configuration();		
		Job job = Job.getInstance(conf);		
		job.setJarByClass(CourseTwo.class);
		job.setMapperClass(MyMapper.class);
		job.setReducerClass(MyReducer.class);	
		job.setMapOutputKeyClass(CourseBean.class);
		job.setMapOutputValueClass(NullWritable.class);
		job.setOutputKeyClass(CourseBean.class);
		job.setOutputValueClass(NullWritable.class);		
		//使用自定义的分区组件
		job.setPartitionerClass(CoursePartitioner.class);
		//和自定义分区组件同时使用，根据分区的个数设置reduceTask的个数
		job.setNumReduceTasks(4);		
		FileInputFormat.setInputPaths(job, new Path(input));
		FileOutputFormat.setOutputPath(job, new Path(output));
		boolean isDone = job.waitForCompletion(true);
		System.exit(isDone ? 0:1);
	}
}

结果：