将数据插入到 hbase 中的方法之 :
BulkLoad 的形式 (公司业务实际生成中使用 ,效率最高)
将静态文件 ===> 生成 Hfile 文件 ===> load 到指定的 hbase 表中(不能设置插入到表的哪个位置)
1 创建 MovieBean ,实现 Writable 接口(序列化) ,重写 write() 和 readFields() 方法
public class MovieBean implements Writable {
private String movie;
private double rate;
private String timeStamp;
private String uid;
public String getMovie() {
return movie;
}
public void setMovie(String movie) {
this.movie = movie;
}
public double getRate() {
return rate;
}
public void setRate(double rate) {
this.rate = rate;
}
public String getTimeStamp() {
return timeStamp;
}
public void setTimeStamp(String timeStamp) {
this.timeStamp = timeStamp;
}
public String getUid() {
return uid;
}
public void setUid(String uid) {
this.uid = uid;
}
@Override
public void write(DataOutput dataOutput) throws IOException { // 重写 write() 方法
dataOutput.writeUTF(movie);
dataOutput.writeDouble(rate);
dataOutput.writeUTF(timeStamp);
dataOutput.writeUTF(uid);
}
@Override
public void readFields(DataInput dataInput) throws IOException { // 重写 readFields() 方法
this.movie = dataInput.readUTF();
this.rate = dataInput.readDouble();
this.timeStamp = dataInput.readUTF();
this.uid = dataInput.readUTF();
}
}
2 将 movie 文件数据一行一行读出来 ,将一行行数据转成一个个 movie 对象 ,进行MR操作后 ,在 reduce 阶段, 将数据转换成Hfile文件 ,将文件导入到hbase的表中
public class LoadTableData {
public static void main(String[] args){
try {
// 获取配置对象
Configuration conf = HBaseConfiguration.create();
// 告知hbase , zookeeper 集群的位置
conf.set("hbase.zookeeper.quorum","doit03:2181,doit04:2181,doit05:2181");
// 获取job对象
Job job = Job.getInstance(conf);
// 设置 mapper 类
job.setMapperClass(MovieMapper.class);
job.setReducerClass(MovieReduce.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(MovieBean.class);
// 设置文件输入路径
FileInputFormat.setInputPaths(job,new Path("D:\\DOITLearning\\10.Hadoop\\mrdata\\movie\\input"));
// reduce将数据转换成Hfile文件 ,将文件导入到hbase的表中
TableMapReduceUtil.initTableReducerJob("hbase01:tb_movie",MovieReduce.class,job);
job.waitForCompletion(true);
} catch (Exception e) {
e.printStackTrace();
}
}
static class MovieMapper extends Mapper<LongWritable,Text, Text,MovieBean>{
Text k = new Text();
Gson gs = new Gson();
@Override
protected void map(LongWritable key, Text value, Context context) {
try {
// 获取一行数据
String line = value.toString();
// 将读取的一行数据转成一个movieBean对象
MovieBean mb = gs.fromJson(line, MovieBean.class);
// 通过mb对象,将movie取出来
String movie = mb.getMovie();
String s = StringUtils.leftPad(movie,6,'0');
String rowkey = s + mb.getTimeStamp();
k.set(rowkey);
context.write(k,mb);
} catch (Exception e) {
e.printStackTrace();
}
}
}
static class MovieReduce extends TableReducer<Text,MovieBean, ImmutableBytesWritable> {
@Override
protected void reduce(Text key, Iterable<MovieBean> values, Context context) {
try {
String rk = key.toString();
// 循环遍历 moviebean
for (MovieBean mb : values) {
// 将一个个movie对象里面的属性取出来 , movie rate timeStemp uid
String movie = mb.getMovie();
double rate = mb.getRate();
String timeStamp = mb.getTimeStamp();
String uid = mb.getUid();
Put put = new Put(Bytes.toBytes(rk));
// 将一个个属性放到列族里面
put.addColumn("cf".getBytes(),"movie".getBytes(),Bytes.toBytes(movie));
put.addColumn("cf".getBytes(),"rate".getBytes(),Bytes.toBytes(rate));
put.addColumn("cf".getBytes(),"timeStamp".getBytes(),Bytes.toBytes(timeStamp));
put.addColumn("cf".getBytes(),"uid".getBytes(),Bytes.toBytes(uid));
// 将数据写出去
context.write(null,put);
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
4 在 hbase shell 客户端查询到的结果如下
flush "hbase01:tb_movie"
000001974055115 column=cf:movie, timestamp=1598825030296, value=1
000001974055115 column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00
000001974055115 column=cf:timeStamp, timestamp=1598825030296, value=974055115
000001974055115 column=cf:uid, timestamp=1598825030296, value=2529
000001974070503 column=cf:movie, timestamp=1598825030296, value=1
000001974070503 column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00
000001974070503 column=cf:timeStamp, timestamp=1598825030296, value=974070503
000001974070503 column=cf:uid, timestamp=1598825030296, value=2516
000001974073729 column=cf:movie, timestamp=1598825030296, value=1
000001974073729 column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00
000001974073729 column=cf:timeStamp, timestamp=1598825030296, value=974073729
000001974073729 column=cf:uid, timestamp=1598825030296, value=2510
000001974076765 column=cf:movie, timestamp=1598825030296, value=1
000001974076765 column=cf:rate, timestamp=1598825030296, value=@\x10\x00\x00\x00\x00\x00\x00
000001974076765 column=cf:timeStamp, timestamp=1598825030296, value=974076765
000001974076765 column=cf:uid, timestamp=1598825030296, value=2507
000001974085111 column=cf:movie, timestamp=1598825030296, value=1
000001974085111 column=cf:rate, timestamp=1598825030296, value=@\x10\x00\x00\x00\x00\x00\x00
000001974085111 column=cf:timeStamp, timestamp=1598825030296, value=974085111
000001974085111 column=cf:uid, timestamp=1598825030296, value=2501
000001974140175 column=cf:movie, timestamp=1598825030296, value=1
000001974140175 column=cf:rate, timestamp=1598825030296, value=@\x08\x00\x00\x00\x00\x00\x00
000001974140175 column=cf:timeStamp, timestamp=1598825030296, value=974140175
000001974140175 column=cf:uid, timestamp=1598825030296, value=2489
000001974146106 column=cf:movie, timestamp=1598825030296, value=1
000001974146106 column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00
000001974146106 column=cf:timeStamp, timestamp=1598825030296, value=974146106
000001974146106 column=cf:uid, timestamp=1598825030296, value=2484
000001974160442 column=cf:movie, timestamp=1598825030296, value=1
000001974160442 column=cf:rate, timestamp=1598825030296, value=@\x08\x00\x00\x00\x00\x00\x00
000001974160442 column=cf:timeStamp, timestamp=1598825030296, value=974160442
000001974160442 column=cf:uid, timestamp=1598825030296, value=2468
000001974168982 column=cf:movie, timestamp=1598825030296, value=1
000001974168982 column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00
000001974168982 column=cf:timeStamp, timestamp=1598825030296, value=974168982
000001974168982 column=cf:uid, timestamp=1598825030296, value=2462
000001974174101 column=cf:movie, timestamp=1598825030296, value=1
000001974174101 column=cf:rate, timestamp=1598825030296, value=@\x10\x00\x00\x00\x00\x00\x00
000001974174101 column=cf:timeStamp, timestamp=1598825030296, value=974174101
000001974174101 column=cf:uid, timestamp=1598825030296, value=2459
000001974175364 column=cf:movie, timestamp=1598825030296, value=1
000001974175364 column=cf:rate, timestamp=1598825030296, value=@\x10\x00\x00\x00\x00\x00\x00
000001974175364 column=cf:timeStamp, timestamp=1598825030296, value=974175364
000001974175364 column=cf:uid, timestamp=1598825030296, value=2858
000001974176741 column=cf:movie, timestamp=1598825030296, value=1
000001974176741 column=cf:rate, timestamp=1598825030296, value=@\x08\x00\x00\x00\x00\x00\x00
.....
.....
.....