Structured Streaming写数据到Hbase中

版本：spark 2.2.0，hadoop 3.0.0，hbase 2.0.0

实现：

1）pom.xml

<dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_${scala.binary.version}</artifactId>
            <version>${spark.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>${hbase.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-log4j12</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
    </dependencies>

2）main

   public static void main(String[] args) throws StreamingQueryException {
        SparkSession sparkSession = SparkSession.builder().appName("test")
                .master("local[*]").getOrCreate();
        Dataset<Row> source = sparkSession
                .readStream()
                .format("socket")
                .option("host", "bigdata-pro01.wn.com")
                .option("port", 9999)
                .load().selectExpr("CAST(value AS STRING)");

        //使用Foreach sink
        StreamingQuery foreachQuery = source.writeStream()
                .foreach(new HbaseSink())
                .outputMode(OutputMode.Append())
                .start();
        foreachQuery.awaitTermination();
    }

3）HbaseSink实现

public class HbaseSink extends ForeachWriter<Row> {
    private Connection connection=null;//hbase连接
    private static Logger logger = LoggerFactory.getLogger(HbaseSink.class);

    @Override
    public boolean open(long partitionId, long version) {
        //hbase
        try{
            Configuration conf = HBaseConfiguration.create();
            conf.set("hbase.zookeeper.property.clientPort", "2181");
            conf.set("hbase.zookeeper.quorum","");
            conf.set("hbase.master", "");
            connection = ConnectionFactory.createConnection(conf);
        }catch(IOException e){
            logger.error(e.toString());
        }
        return true;
    }

    @Override
    public void process(Row value) {
        String word = value.getAs("value").toString();
        String rowkey = String.valueOf(System.currentTimeMillis());//以当前时间戳作为rowkey
        String columnFamily = "info";//列簇
        Put put = new Put(rowkey.getBytes());
        put.addColumn(columnFamily.getBytes(), ("word").getBytes(), word.getBytes());
        try{
            TableName tableName = TableName.valueOf("test");//表名
            Table table = connection.getTable(tableName);
            table.put(put);
        }catch(IOException e){
            e.printStackTrace();
        }
    }

    @Override
    public void close(Throwable errorOrNull) {
        try{
            connection.close();
        }catch(IOException e){
            e.printStackTrace();
        }
    }
}