spark读写hbase

/**
    * spark从hbase中读取数据
    * @param hbaseConfig
    * @param spark
    */
  def getHbase(hbaseConfig:Configuration,spark:SparkSession): Unit ={
      hbaseConfig.set(TableInputFormat.INPUT_TABLE,"stu_person") ;
      val hbaseRdd = spark.sparkContext.newAPIHadoopRDD(hbaseConfig,classOf[TableInputFormat],
      classOf[org.apache.hadoop.hbase.io.ImmutableBytesWritable],
      classOf[org.apache.hadoop.hbase.client.Result]);
    hbaseRdd.foreach(result=>{
      val key = Bytes.toString(result._2.getRow);
      val deptid = Bytes.toString(result._2.getValue("info".getBytes,"deptid".getBytes));
      val name = Bytes.toString(result._2.getValue("info".getBytes,"name".getBytes));
      val salary = Bytes.toString(result._2.getValue("info".getBytes,"salary".getBytes));
      println("key:"+key+":info{name:"+name+"}");
    });
  }

 /**
    * spark写入数据到hbase
    * @param hbaseConfig
    * @param spark
    */
  def writeHbase(hbaseConfig:Configuration,spark:SparkSession): Unit ={
    val jobConf = new JobConf(hbaseConfig);
    jobConf.setOutputFormat(classOf[TableOutputFormat]) ;
    jobConf.set(TableOutputFormat.OUTPUT_TABLE,"stu_person");
    val array = Array("002,dept-1,lee,2300",
                      "002,dept-1,lily,3300",
                      "003,dept-2,lucy,4300",
                      "003,dept-2,chen,2100",
                      "004,dept-3,sun,5300",
                      "004,dept-3,hello,9300");
    val rdd = spark.sparkContext.makeRDD(array) ;
    val resultRDD = rdd.map(line=>line.split(",")).map(x=>{
      val put = new Put(Bytes.toBytes(x(0)));
      put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("deptid"),Bytes.toBytes(x(1))) ;
      put.addColumn(Bytes.toBytes("info"),Bytes.toBytes("name"),Bytes.toBytes(x(2))) ;
      put.addColumn(Bytes.toBytes("info"), Bytes.toBytes("salary"), Bytes.toBytes(x(3)));
      (new ImmutableBytesWritable,put)
    });
    resultRDD.saveAsHadoopDataset(jobConf) ;
  }

 

转载于:https://my.oschina.net/maokukude/blog/3045632