Hbase之 java 客户端上 table 数据的插入( BulkLoad )之 movie 案例(8)

将数据插入到 hbase 中的方法之 :
BulkLoad 的形式 (公司业务实际生成中使用 ,效率最高)
将静态文件 ===> 生成 Hfile 文件 ===> load 到指定的 hbase 表中(不能设置插入到表的哪个位置)

1 创建 MovieBean ,实现 Writable 接口(序列化) ,重写 write() 和 readFields() 方法

public class MovieBean implements Writable {
    private String movie;
    private double rate;
    private String timeStamp;
    private String uid;

    public String getMovie() {
        return movie;
    }

    public void setMovie(String movie) {
        this.movie = movie;
    }

    public double getRate() {
        return rate;
    }

    public void setRate(double rate) {
        this.rate = rate;
    }

    public String getTimeStamp() {
        return timeStamp;
    }

    public void setTimeStamp(String timeStamp) {
        this.timeStamp = timeStamp;
    }

    public String getUid() {
        return uid;
    }

    public void setUid(String uid) {
        this.uid = uid;
    }
    @Override
    public void write(DataOutput dataOutput) throws IOException {                       //  重写 write() 方法
        dataOutput.writeUTF(movie);
        dataOutput.writeDouble(rate);
        dataOutput.writeUTF(timeStamp);
        dataOutput.writeUTF(uid);
    }
    @Override
    public void readFields(DataInput dataInput) throws IOException {               //  重写 readFields() 方法
        this.movie = dataInput.readUTF();
        this.rate = dataInput.readDouble();
        this.timeStamp = dataInput.readUTF();
        this.uid = dataInput.readUTF();
    }
}

2 将 movie 文件数据一行一行读出来 ,将一行行数据转成一个个 movie 对象 ,进行MR操作后 ,在 reduce 阶段, 将数据转换成Hfile文件 ,将文件导入到hbase的表中

public class LoadTableData {
    public static void main(String[] args){
        try {
            // 获取配置对象
            Configuration conf = HBaseConfiguration.create();
            // 告知hbase , zookeeper 集群的位置
            conf.set("hbase.zookeeper.quorum","doit03:2181,doit04:2181,doit05:2181");
            // 获取job对象
            Job job = Job.getInstance(conf);
            // 设置 mapper 类
            job.setMapperClass(MovieMapper.class);
            job.setReducerClass(MovieReduce.class);
            job.setMapOutputKeyClass(Text.class);
            job.setMapOutputValueClass(MovieBean.class);

            // 设置文件输入路径
            FileInputFormat.setInputPaths(job,new Path("D:\\DOITLearning\\10.Hadoop\\mrdata\\movie\\input"));

            // reduce将数据转换成Hfile文件 ,将文件导入到hbase的表中
            TableMapReduceUtil.initTableReducerJob("hbase01:tb_movie",MovieReduce.class,job);
            job.waitForCompletion(true);
        } catch (Exception e) {
            e.printStackTrace();
        }

    }
    static class MovieMapper extends Mapper<LongWritable,Text, Text,MovieBean>{
        Text k = new Text();
        Gson gs = new Gson();
        @Override
        protected void map(LongWritable key, Text value, Context context) {
            try {
                // 获取一行数据
                String line = value.toString();
                // 将读取的一行数据转成一个movieBean对象
                MovieBean mb = gs.fromJson(line, MovieBean.class);
                // 通过mb对象,将movie取出来
                String movie = mb.getMovie();
                String s = StringUtils.leftPad(movie,6,'0');
                String rowkey = s + mb.getTimeStamp();
                k.set(rowkey);
                context.write(k,mb);
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
    static class MovieReduce extends TableReducer<Text,MovieBean, ImmutableBytesWritable> {
        @Override
        protected void reduce(Text key, Iterable<MovieBean> values, Context context) {
            try {
                String rk = key.toString();
                // 循环遍历 moviebean 
                for (MovieBean mb : values) {
                //  将一个个movie对象里面的属性取出来 , movie   rate   timeStemp   uid
                    String movie = mb.getMovie();
                    double rate = mb.getRate();
                    String timeStamp = mb.getTimeStamp();
                    String uid = mb.getUid();
                    
                    
                    Put put = new Put(Bytes.toBytes(rk));
                    //  将一个个属性放到列族里面
                    put.addColumn("cf".getBytes(),"movie".getBytes(),Bytes.toBytes(movie));
                    put.addColumn("cf".getBytes(),"rate".getBytes(),Bytes.toBytes(rate));
                    put.addColumn("cf".getBytes(),"timeStamp".getBytes(),Bytes.toBytes(timeStamp));
                    put.addColumn("cf".getBytes(),"uid".getBytes(),Bytes.toBytes(uid));
                    // 将数据写出去
                    context.write(null,put);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}

4 在 hbase shell 客户端查询到的结果如下

flush "hbase01:tb_movie"
 000001974055115               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974055115               column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00            
 000001974055115               column=cf:timeStamp, timestamp=1598825030296, value=974055115                           
 000001974055115               column=cf:uid, timestamp=1598825030296, value=2529                                      
 000001974070503               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974070503               column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00            
 000001974070503               column=cf:timeStamp, timestamp=1598825030296, value=974070503                           
 000001974070503               column=cf:uid, timestamp=1598825030296, value=2516                                      
 000001974073729               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974073729               column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00            
 000001974073729               column=cf:timeStamp, timestamp=1598825030296, value=974073729                           
 000001974073729               column=cf:uid, timestamp=1598825030296, value=2510                                      
 000001974076765               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974076765               column=cf:rate, timestamp=1598825030296, value=@\x10\x00\x00\x00\x00\x00\x00            
 000001974076765               column=cf:timeStamp, timestamp=1598825030296, value=974076765                           
 000001974076765               column=cf:uid, timestamp=1598825030296, value=2507                                      
 000001974085111               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974085111               column=cf:rate, timestamp=1598825030296, value=@\x10\x00\x00\x00\x00\x00\x00            
 000001974085111               column=cf:timeStamp, timestamp=1598825030296, value=974085111                           
 000001974085111               column=cf:uid, timestamp=1598825030296, value=2501                                      
 000001974140175               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974140175               column=cf:rate, timestamp=1598825030296, value=@\x08\x00\x00\x00\x00\x00\x00            
 000001974140175               column=cf:timeStamp, timestamp=1598825030296, value=974140175                           
 000001974140175               column=cf:uid, timestamp=1598825030296, value=2489                                      
 000001974146106               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974146106               column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00            
 000001974146106               column=cf:timeStamp, timestamp=1598825030296, value=974146106                           
 000001974146106               column=cf:uid, timestamp=1598825030296, value=2484                                      
 000001974160442               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974160442               column=cf:rate, timestamp=1598825030296, value=@\x08\x00\x00\x00\x00\x00\x00            
 000001974160442               column=cf:timeStamp, timestamp=1598825030296, value=974160442                           
 000001974160442               column=cf:uid, timestamp=1598825030296, value=2468                                      
 000001974168982               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974168982               column=cf:rate, timestamp=1598825030296, value=@\x14\x00\x00\x00\x00\x00\x00            
 000001974168982               column=cf:timeStamp, timestamp=1598825030296, value=974168982                           
 000001974168982               column=cf:uid, timestamp=1598825030296, value=2462                                      
 000001974174101               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974174101               column=cf:rate, timestamp=1598825030296, value=@\x10\x00\x00\x00\x00\x00\x00            
 000001974174101               column=cf:timeStamp, timestamp=1598825030296, value=974174101                           
 000001974174101               column=cf:uid, timestamp=1598825030296, value=2459                                      
 000001974175364               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974175364               column=cf:rate, timestamp=1598825030296, value=@\x10\x00\x00\x00\x00\x00\x00            
 000001974175364               column=cf:timeStamp, timestamp=1598825030296, value=974175364                           
 000001974175364               column=cf:uid, timestamp=1598825030296, value=2858                                      
 000001974176741               column=cf:movie, timestamp=1598825030296, value=1                                       
 000001974176741               column=cf:rate, timestamp=1598825030296, value=@\x08\x00\x00\x00\x00\x00\x00
 .....
 .....
 .....