pyspark连接hbase,并将spark-rdd转化为dataframe@TOC
建立spark连接,获取rdd
#-*- coding:utf-8 -*-
import json
from pyspark.sql import SparkSession
host = '192.168.11.xxx'
#table name
table = 'I_OCS_COLLECT'
#建立spark连接
spark = SparkSession.builder.master("yarn-client").appName("test").getOrCreate()
hbaseconf = {"hbase.zookeeper.quorum": host,
"hbase.mapreduce.inputtable": table
#定义起止行
#"hbase.mapreduce.scan.row.start": row,
# "hbase.mapreduce.scan.row.stop": row1
}
keyConv = "org.apache.spark.examples.pythonconverters.ImmutableBytesWritableToStringConverter"
valueConv = "org.apache.spark.examples.pythonconverters.HBaseResultToStringConverter"
#得到rdd
hbase_rdd = spark.sparkContext.newAPIHadoopRDD("org.apache.hadoop.hbase.mapreduce.TableInputFormat",
"org.apache.hadoop.hbase.io.ImmutableBytesWritable",
"org.apache.hadoop.hbase.client.Result",
keyConverter=keyConv, valueConverter=valueConv, conf=hbaseconf)
数据处理
定义函数deal_missing_dec,定义不能为空的列
def deal_missing_dec(no_row_key_colnames):
def deal_missing(x):
result = {}
for i in no_row_key_colnames:
if i in x[1].keys():
result[i]=x[1][i]
else:
result[i]='missing'
return (x[0],result)
return deal_missing
将rdd转化为dataframe
def deal_row(x):
return [x[0]]+list(x[1].values())
def rdd_to_df(hbase_rdd):
data_split = hbase_rdd.map(lambda x:(x[0],x[1].split('\n')))
data_cols = data_split.map(lambda x:(x[0],call_transfor(x[1])))
no_row_key_colnames = data_cols.map(lambda x:[i for i in x[1]]).take(2)[1]
deal_missing = deal_missing_dec(no_row_key_colnames)
no_missing = data_cols.map(deal_missing)
data = no_missing.map(deal_row).toDF(['row_key']+no_row_key_colnames)
return data
调用
fdc_data = rdd_to_df(hbase_rdd)
print fdc_data.show()
结果:
参考文章:
https://blog.csdn.net/hchzhao_1985/article/details/82717949
http://dblab.xmu.edu.cn/blog/1715-2/