在使用MapReduce对hbase中的数据进行分析时,通过编写MapReduce程序,对数据进行分析,当我使用MapReduce进行分析后的结果存储到mysql等关系型数据库。
在对hbase中的数据仅分析时,要求算出每人每年的通话时长,每月的通过时常,每天的通话时常,和通话次数,是通过一个MapReduce进行编写的,然后当想要把数据存储到3张表时,遇到了困难,以前存一张表时都是使用的DBInputFormat这个类,但是这个类没有找到存储多张表的方法,所以就使用了自定义类型的outputFormat,把mysql的数据存储到多张表中,然后进行了测试成功了
1、继承RecordWriter
package com.HbaseMR;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import java.io.IOException;
import java.sql.*;
//此处的泛型是reduce端的输出的类型的key、 value
public class MSRecordWrite extends RecordWriter<CallDate, Text> {
private static Connection conn ;
static {
String driver = "com.mysql.jdbc.Driver";
String url = "jdbc:mysql://localhost:3306/hbase2db";
String user = "root";
String password = "root";
try {
Class.forName(driver);
conn = DriverManager.getConnection(url, user, password);
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void write(CallDate key, Text value) throws IOException, InterruptedException {
// TODO Auto-generated method stub
System.out.println("start to write" + value.toString());
try {
System.out.println("it already gets into insert function");
PreparedStatement pstmt = null;
final String callPhone = key.getCallPhone();
final String callDate = key.getCallDate();
final String[] split = value.toString().split("\t");
long callTime = Integer.parseInt(split[0]);
int counts = Integer.parseInt(split[1]);
//年份
if (callDate.length()==4){
pstmt = conn.prepareStatement("insert into call_data_y values(?,?,?,?)");
}else if(callDate.length() ==7){
//按月份
pstmt = conn.prepareStatement("insert into call_data_m values(?,?,?,?)");
}else {
//按天
pstmt = conn.prepareStatement("insert into call_data_d values(?,?,?,?)");
}
//进行赋值
pstmt.setString(1,callPhone);
pstmt.setString(2,callDate);
pstmt.setLong(3,callTime);
pstmt.setInt(4,counts);
pstmt.executeUpdate();
pstmt.close();
System.out.println("insert is successfuly " + value.toString());
} catch (Exception ex) {
try {
conn.close();
} catch (SQLException e) {
e.printStackTrace();
}
ex.printStackTrace();
}
}
@Override
public void close(TaskAttemptContext context) throws IOException, InterruptedException {
}
}
2、继承OutputFormat类