用python实现Hbase的读写操作

读取Habse中的数据

from pyspark import SparkContext, SparkConf

# spark集群的地址
conf = SparkConf().setMaster('local').setAppName('ReadHbase')
sc = SparkContext(conf=conf)

# 配置 连接参数
host = 'localhost'
table = 'student'
conf = {"hbase.zookeeper.quorum": host, "hbase.mapreduce.inputtable": table}

# 键转化器 将键值对的格式转化为字符串的格式
keyConv = "org.apache.spark.examples.pythonconverters.ImmutableBytesWritableToStringConverter"
# 值转换器
valueConv = "org.apache.spark.examples.pythonconverters.HBaseResultToStringConverter"

# pysaprk连接hbase目前只能通过sparkContext对象，所以这里用的前面设置的spark_context
hbase_rdd = sc.newAPIHadoopRDD("org.apache.hadoop.hbase.mapreduce.TableInputFormat",
                                "org.apache.hadoop.hbase.io.ImmutableBytesWritable",
                                "org.apache.hadoop.hbase.client.Result",
                                keyConverter=keyConv,
                                valueConverter=valueConv,
                                conf=conf)


# 如果想通过sparkSesssion对象连接hbase，代码如下
'''
 hbase_rdd = spark_session.saprkContext.newAPIHadoopRDD("org.apache.hadoop.hbase.mapreduce.TableInputFormat",
                                            "org.apache.hadoop.hbase.io.ImmutableBytesWritable",
                                            "org.apache.hadoop.hbase.client.Result",
                                             keyConverter=keyConv,
                                             valueConverter=valueConv,                                             
'''
# 统计 多少个行键
count = hbase_rdd.count()
# 进行缓存 存储
hbase_rdd.cache()
# collect的作用是 将键值对分装到一个列表进行返回
out_put = hbase_rdd.collect()
# 将内容以键值对的方式读出来
for (k, v) in out_put:
    print(k, v)

将数据写入Hbase中

from pyspark import SparkContext, SparkConf

conf = SparkConf().setMaster('local').setAppName('ReadHbase')
sc = SparkContext(conf=conf)
host = "localhost"
table = "student"
keyConv = "org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter"
valueConv = "org.apache.spark.examples.pythonconverters.StringListToPutConverter"
conf = {"hbase.zookeeper.quorum": host, "hbase.mapred.outputtable": table,
        "mapreduce.outputformat.class": "org.apache.hadoop.hbase.mapreduce.TableOutputFormat",
        "mapreduce.job.output.key.class": "org.apache.hadoop.hbase.io.ImmutableBytesWritable",
        "mapreduce.job.output.value.class": "org.apache.hadoop.io.Writable"}

rawData = ['3,info,name,huangxingxing', '4,info,name,zhangshuaijun']
# 先生成RDD x指的是3,info,name,huangxingxing,x[0] = 3指的是行键,x.split(",")后得到一个列表：[3,info,name,huangxingxing];
# .map()之后 将这些字符串列表转化为{key:values} {row key:3,info,name,huangxingxing}
sc.parallelize(rawData).map(lambda x: (x[0], x.split(","))).saveAsHadoopDataset(conf=conf, keyConverter=keyConv, valueConverter=valueConv)