Hadoop-MapReduce基本代码实现步骤

JavaBean类

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

public class Student implements WritableComparable<Student> {
    private String sid;
    private String sex;
    private String sname;
    private Integer age;
    private String classes;
    private String year;
    private Integer chinese;
    private Integer math;
    private Integer english;

    @Override
    public String toString() {
        return sid + " " + sex + " " + sname + " " + age + " " + classes + " " + year;
    }

    public Student() {
    }

    public Student(String sid, String sex, String sname, Integer age, String classes, String year, Integer chinese, Integer math, Integer english) {
        this.sid = sid;
        this.sex = sex;
        this.sname = sname;
        this.age = age;
        this.classes = classes;
        this.year = year;
        this.chinese = chinese;
        this.math = math;
        this.english = english;
    }

    public String getSid() {
        return sid;
    }

    public void setSid(String sid) {
        this.sid = sid;
    }

    public String getSex() {
        return sex;
    }

    public void setSex(String sex) {
        this.sex = sex;
    }

    public String getSname() {
        return sname;
    }

    public void setSname(String sname) {
        this.sname = sname;
    }

    public Integer getAge() {
        return age;
    }

    public void setAge(Integer age) {
        this.age = age;
    }

    public String getClasses() {
        return classes;
    }

    public void setClasses(String classes) {
        this.classes = classes;
    }

    public String getYear() {
        return year;
    }

    public void setYear(String year) {
        this.year = year;
    }

    public Integer getChinese() {
        return chinese;
    }

    public void setChinese(Integer chinese) {
        this.chinese = chinese;
    }

    public Integer getMath() {
        return math;
    }

    public void setMath(Integer math) {
        this.math = math;
    }

    public Integer getEnglish() {
        return english;
    }

    public void setEnglish(Integer english) {
        this.english = english;
    }

    @Override
    public int compareTo(Student o) {
        return this.age < o.getAge() ? -1 : 1;
    }

    @Override
    public void write(DataOutput dataOutput) throws IOException {
        dataOutput.writeUTF(sid);
        dataOutput.writeUTF(sex);
        dataOutput.writeUTF(sname);
        dataOutput.writeInt(age);
        dataOutput.writeUTF(classes);
        dataOutput.writeUTF(year);
        dataOutput.writeInt(chinese);
        dataOutput.writeInt(math);
        dataOutput.writeInt(english);
    }

    @Override
    public void readFields(DataInput dataInput) throws IOException {
        this.sid = dataInput.readUTF();
        this.sex = dataInput.readUTF();
        this.sname = dataInput.readUTF();
        this.age = dataInput.readInt();
        this.classes = dataInput.readUTF();
        this.year = dataInput.readUTF();
        this.chinese = dataInput.readInt();
        this.math = dataInput.readInt();
        this.english = dataInput.readInt();
    }
}

Map类

import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class WordCountMap extends Mapper<LongWritable, Text,Student, NullWritable> {
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        String[] split = value.toString().split("\\t");
        context.write(new Student(split[0],split[1],split[2],Integer.parseInt(split[3]),split[4],split[5],Integer.parseInt(split[6]),Integer.parseInt(split[7]),Integer.parseInt(split[8])),NullWritable.get());
    }
}

Partitions类

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Partitioner;

public class Partitions  extends Partitioner<Student, NullWritable> {
    @Override
    public int getPartition(Student student, NullWritable nullWritable, int i) {
        if (student.getClasses().startsWith("一")){
            return 0;
        }else if (student.getClasses().startsWith("二")){
            return 1;
        }else if (student.getClasses().startsWith("三")){
            return 2;
        }else if (student.getClasses().startsWith("四")){
            return 3;
        }else {
            return 4;
        }
    }
}

Reduce类

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class WordCountReduce extends Reducer<Student, NullWritable,Student, Text> {
    @Override
    protected void reduce(Student key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
        context.write(key,new Text("平均成绩: "+(key.getChinese()+key.getMath()+key.getEnglish())/3));
    }
}

Driver类

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCountDriver {
    public static void main(String[] args) throws Exception {
        Job job = Job.getInstance(new Configuration());

        job.setJarByClass(WordCountDriver.class);
        job.setMapperClass(WordCountMap.class);
        job.setReducerClass(WordCountReduce.class);

        job.setMapOutputKeyClass(Student.class);
        job.setMapOutputValueClass(NullWritable.class);

        job.setOutputKeyClass(Student.class);
        job.setOutputValueClass(Text.class);

        job.setPartitionerClass(Partitions.class);
        job.setNumReduceTasks(5);


        FileInputFormat.addInputPath(job,new Path("/student_score.txt"));
        FileOutputFormat.setOutputPath(job,new Path(args[0]));

        boolean b = job.waitForCompletion(true);
        System.exit(b?0:1);


    }
}