在spark中采用sc.hadoopConfiguration进行数据传输java.lang.IllegalArgumentException: Can not create a Path from a null string
scala 2.11
spark 2.20
这是spark自身一些bug
https://issues.apache.org/jira/browse/SPARK-21549
如何解决:
spark2.2
源码
def rddWriteToHbase(tableName:String,sc:SparkContext,rdd:RDD[(ImmutableBytesWritable,org.apache.hadoop.hbase.client.Put)])={
val conf = HBaseConfiguration.create()
val usertable=TableName.valueOf(tableName)
var admin = ConnectionFactory.createConnection(conf).getAdmin
if (admin.tableExists(usertable)){
println("this table has already exists!!!")
} else {
val tableDescr=new HTableDescriptor(usertable)
val FamilyName=new HColumnDescriptor("cf".getBytes)
// FamilyName.setCompressionType(Algorithm.SNAPPY)
tableDescr.addFamily(FamilyName)
admin.createTable(tableDescr)
}
sc.hadoopConfiguration.set(TableOutputFormat.OUTPUT_TABLE, tableName)
sc.hadoopConfiguration.set("mapreduce.output.fileoutputformat.outputdir", "/tmp")
val job = Job.getInstance(sc.hadoopConfiguration)
job.setOutputKeyClass(classOf[ImmutableBytesWritable])
job.setOutputValueClass(classOf[Result])
job.setOutputFormatClass(classOf[TableOutputFormat[ImmutableBytesWritable]])
rdd.saveAsNewAPIHadoopDataset(job.getConfiguration)
// var jobConf = new JobConf(HBaseConfiguration.create)
// jobConf.set(TableOutputFormat.OUTPUT_TABLE, "window2")
// jobConf.setOutputFormat(classOf[TableOutputFormat])//不加这句会报错Undefined job output-path
// //在JobConf中,通常需要关注或者设置五个参数
// //文件的保存路径、key值的class类型、value值的class类型、RDD的输出格式(OutputFormat)、以及压缩相关的参数
//
// rdd.saveAsHadoopDataset(jobConf)
}
//IMPORTANT: must set the attribute to solve the problem (can't create path from null string )
sc.hadoopConfiguration.set("mapreduce.output.fileoutputformat.outputdir", "/tmp")
第二种:
采用spark2.1版本或者升高版本2.3即可