Spark之RDD输出到不同的文件名和目录

import org.apache.hadoop.io.NullWritable import org.apache.hadoop.mapred.lib.MultipleTextOutputFormat /** * ClassName: qqqq * Author: yage.liu * Date: 2017年12月01日 17:02 * Version: V1.0 * Description: **/ class RDDMultipleTextOutputFormat extends MultipleTextOutputFormat[Any, Any] { //指定该条记录的输出文件 override def generateFileNameForKeyValue(key: Any, value: Any, name: String): String ={ //第1种：输出多个目录 //(key + "/" + name) //第2种：数据多个文件 key.asInstanceOf[String] //第3种： /*if(key.toString.contains(ETLConfig.ERROR_SEPARATOR)){ ETLConfig.ERROR_SEPARATOR.toUpperCase }else{ key.asInstanceOf[String] }*/ } //输出时不输出key override def generateActualKey(key: Any, value: Any): Any = NullWritable.get() }

outputRDD.partitionBy(new HashPartitioner(minPartitions)).saveAsHadoopFile(targetTable, String.class, String.class,RDDMultipleTextOutputFormat.class);