MapReduce之自定义分区
0.概括
- 要求:按照员工的部门号分为3个分区
- 思路:
1、定义一个java类EmpPartitioner,实现Partitioner接口
2、重写方法getPartition,编写分区规则
3、只需写Mapper阶段,验证是否按照分区规则进行分区
4、编写Job类,设置mapper及输入输出
1.在pom.xml中添加hadoop依赖
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.7.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>2.7.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
<version>2.7.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>2.7.3</version>
</dependency>
2.依赖
log4j.rootLogger=FATAL, dest1
log4j.logger.dsaLogging=DEBUG, dsa
log4j.additivity.dsaLogging=false
log4j.appender.dest1=org.apache.log4j.ConsoleAppender
log4j.appender.dest1.layout=org.apache.log4j.PatternLayout
log4j.appender.dest1.layout.ConversionPattern=%-5p:%l: %m%n
log4j.appender.dest1.ImmediateFlush=true
log4j.appender.dsa=org.apache.log4j.RollingFileAppender
log4j.appender.dsa.File=./logs/dsa.log
log4j.appender.dsa.MaxFileSize=2000KB
# Previously MaxBackupIndex=2
log4j.appender.dsa.MaxBackupIndex=5
log4j.appender.dsa.layout=org.apache.log4j.PatternLayout
log4j.appender.dsa.layout.ConversionPattern=%l:%d: %m%n
3.编写Employee序列化类实现WritableComparable接口
package com.mr.partition;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
public class Employee implements WritableComparable<Employee> {
//7369,SMITH,CLERK,7902,1980/12/17,800,,20
private IntWritable empNo;
private Text empName;
private Text empJob;
private IntWritable leaderNo;
private Text hireDate;
private IntWritable empSalary;
private Text empBonus;
private IntWritable deptNo;
public Employee() {
this.empNo = new IntWritable();
this.empName = new Text("");
this.empJob = new Text("");
this.leaderNo = new IntWritable();
this.hireDate = new Text("");
this.empSalary =new IntWritable();
this.empBonus = new Text("");
this.deptNo = new IntWritable();
}
public Employee(int empNo, String empName, String empJob, int leaderNo,
String hireDate, int empSalary, String empBonus, int deptNo) {
this.empNo = new IntWritable(empNo);
this.empName = new Text(empName);
this.empJob = new Text(empJob);
this.leaderNo = new IntWritable(leaderNo);
this.hireDate = new Text(hireDate);
this.empSalary =new IntWritable(empSalary);
this.empBonus = new Text(empBonus);
this.deptNo = new IntWritable(deptNo);
}
@Override
public void write(DataOutput out) throws IOException {
//序列化
this.empNo.write(out);
this.empName.write(out);
this.empJob.write(out);
this.leaderNo.write(out);
this.hireDate.write(out);
this.empSalary.write(out);
this.empBonus.write(out);
this.deptNo.write(out);
}
@Override
public void readFields(DataInput in) throws IOException {
this.empNo.readFields(in);
this.empName.readFields(in);
this.empJob.readFields(in);
this.leaderNo.readFields(in);
this.hireDate.readFields(in);
this.empSalary.readFields(in);
this.empBonus.readFields(in);
this.deptNo.readFields(in);
}
@Override
public String toString() {
return "Employee{" +
"empNo=" + empNo +
", empName=" + empName +
", empJob=" + empJob +
", leaderNo=" + leaderNo +
", hireDate=" + hireDate +
", empSalary=" + empSalary +
", empBonus=" + empBonus +
", deptNo=" + deptNo +
'}';
}
public IntWritable getEmpNo() {
return empNo;
}
public void setEmpNo(IntWritable empNo) {
this.empNo = empNo;
}
public Text getEmpName() {
return empName;
}
public void setEmpName(Text empName) {
this.empName = empName;
}
public Text getEmpJob() {
return empJob;
}
public void setEmpJob(Text empJob) {
this.empJob = empJob;
}
public IntWritable getLeaderNo() {
return leaderNo;
}
public void setLeaderNo(IntWritable leaderNo) {
this.leaderNo = leaderNo;
}
public Text getHireDate() {
return hireDate;
}
public void setHireDate(Text hireDate) {
this.hireDate = hireDate;
}
public IntWritable getEmpSalary() {
return empSalary;
}
public void setEmpSalary(IntWritable empSalary) {
this.empSalary = empSalary;
}
public Text getEmpBonus() {
return empBonus;
}
public void setEmpBonus(Text empBonus) {
this.empBonus = empBonus;
}
public IntWritable getDeptNo() {
return deptNo;
}
public void setDeptNo(IntWritable deptNo) {
this.deptNo = deptNo;
}
/**
* 自定义排序规则
* 按照部门号升序排
* @param o
* @return
*/
// public int compareTo(Employee o) {
// if (this.deptNo.get() > o.getDeptNo().get()){
// return 1;
// }else if(this.deptNo.get() < o.getDeptNo().get()){
// return -1;
// }else{
// return 0;
// }
// }
/**
* 自定义排序规则
* 按照部门号升序排,员工工资降序排序
* @param o
* @return
*/
public int compareTo(Employee o) {
if (this.deptNo.get() > o.getDeptNo().get()){
return 1;
}else if(this.deptNo.get() < o.getDeptNo().get()){
return -1;
}
//说明:部门号是相同的情况,执行下面代码
//按照工资降序排
if (this.empSalary.get() > o.getEmpSalary().get()){
return -1;
}else if(this.empSalary.get() < o.getEmpSalary().get()){
return 1;
}else{
return 0;
}
}
}
4.编写分区规则类EmpPartitioner
package com.mr.partition;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Partitioner;
//key2 value2
public class EmpPartitioner extends Partitioner<Employee, IntWritable> {
/**
* 按照员工所属部分来进行分区:
* 条件1:10部门员工放在1号区
* 条件2:20部门员工放在2号区
* 条件3:30部门员工放在0号区
* @param employee
* @param intWritable
* @param numPartitions
* @return
*/
public int getPartition(Employee employee, IntWritable intWritable, int numPartitions) {
if(10 == employee.getDeptNo().get()){
System.out.println("===========getPartition 10 ==============");
return 1 % numPartitions;
}
if(20 == employee.getDeptNo().get()){
System.out.println("===========getPartition 20 ==============");
return 2 % numPartitions;
}
if(30 == employee.getDeptNo().get()){
System.out.println("===========getPartition 30==============");
return 3 % numPartitions;
}
//上述条件不满足情况下,给定一个分区
return 3;
}
}
5.编写mapper类
package com.mr.partition;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import java.io.IOException;
//key1 value1 key2 value2
public class EmpMapper extends Mapper<LongWritable, Text, Employee, IntWritable> {
//保证key的一致性
Employee employee = new Employee();
protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
//数据格式:<0,7369,SMITH,CLERK,7902,1980/12/17,800,,20>
//1、分词
String[] splits = value.toString().split(",");
//2、创建Employee对象,并且赋值
employee.setDeptNo(new IntWritable(Integer.valueOf(splits[7])));
employee.setEmpSalary(new IntWritable(Integer.valueOf(splits[5])));
//3、通过context写出去
context.write(employee, employee.getEmpSalary());
}
}
6.编写Job类
package com.mr.partition;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import java.util.Random;
public class EmpJob {
public static void main(String[] args) throws Exception {
Job job = Job.getInstance(new Configuration());
job.setJarByClass(EmpJob.class);
job.setMapperClass(EmpMapper.class);
job.setMapOutputKeyClass(Employee.class);//key2 : Employee
job.setMapOutputValueClass(IntWritable.class);//value2
job.setOutputKeyClass(Employee.class);
job.setOutputValueClass(IntWritable.class);
//设置分区类及分区数
job.setPartitionerClass(EmpPartitioner.class);
job.setNumReduceTasks(3);//设置3个reduce task (分区数)
//先使用本地文件做测试
FileInputFormat.setInputPaths(job,new Path("D:\\emp.csv"));
FileOutputFormat.setOutputPath(job,new Path(getOutputDir()));
boolean result = job.waitForCompletion(true);
System.out.println("result:" + result);
}
//用于产生随机输出目录
public static String getOutputDir(){
String prefix = "D:\\output\\";
long time = System.currentTimeMillis();
int random = new Random().nextInt();
return prefix + "result_" + time + "_" + random;
}
}
7.运行
分区实现