java合并spark小文件_spark小文件合并解决多级分区

package spark10.access

import java.text.SimpleDateFormat

import java.util.Date

import org.apache.hadoop.fs.{FileSystem, Path}

import org.apache.log4j.Logger

import org.apache.spark.sql.hive.HiveContext

import org.apache.spark.{SparkConf, SparkContext, sql}

import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode}

/**

* Created by hadoop on 下午10:01.

*

*/

object AccesslogETL {

def main(args: Array[String]): Unit = {

val sc = new SparkContext(new SparkConf().setAppName("access").setMaster("local[3]"))

val sqlContext = new HiveContext(sc)

// doJob(sqlContext, fileSystem, appName, inputPath, outputPath,

// loadTime, accessTable, ifRefreshPartiton, tryTime)

}

/**

*

* @param sqlContext

* @param fileSystem

* @param batchID batchID, 格式:201803221505-g1

* @param inputPath

* @param outputPath

* @param accessTable 清洗后外部表

* @param ifRefreshPartiton 是否刷新分区: 0-不刷新, 1-刷新

* @param tryTime 重试次数

* @return

*/

@throws(classOf[Exception])

def doJob(sqlContext: SQLContext, fileSystem: FileSystem, batchID: String,

inputPath: String, outputPath: String, dataTime:String,

accessTable: String, ifRefreshPartiton: String, tryTime: Int): String = {

try{

var result = "batchID:" + batchID

val logger = Logger.getLogger("org")

var begin = new Date().getTime

val inputLocation = inputPath + batchID

val inputDoingLocation = inputPath + "/" + batchID + "_doing"

val dirExists = fileSystem.exists(new Path(inputLocation))

if (!dirExists && tryTime == 0) {

logger.info(s"$inputLocation not exists")

result = result + s" $inputLocation not exists"

return result

} else if (!dirExists && tryTime > 0) {

if (!fileSystem.exists(new Path(inputDoingLocation))) {

logger.info(s" $inputDoingLocation not exists")

result = result + s" $inputDoingLocation not exists"

return result

}

} else {

val isDoingRenamed = renameHDFSDir(fileSystem, inputLocation, inputDoingLocation)

if (!isDoingRenamed) {

logger.info(s" $inputLocation move to $inputDoingLocation failed")

result = result + s" $inputLocation move to $inputDoingLocation failed"

return result

}

logger.info(s" $inputLocation move to $inputDoingLocation success")

}

val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")

val endTime = sdf.parse(dataTime)

val beginTime = sdf.format(endTime.getTime() - 2*60*60*1000)

val curHourTime = sdf.format(endTime.getTime() - 1*60*60*1000)

var resultDF:DataFrame = null

// 遍历目录下的所有house子目录

fileSystem.globStatus(new Path(inputDoingLocation+"/*")).foreach(p=>{

val hLoc = p.getPath.toString

// 调用计算分区大小的代码

val getPartitionNum = (houseLoc:String)=>{

//计算逻辑略

1

}

val partitionNum = getPartitionNum(hLoc)

logger.info("hLoc:" + hLoc + ", partitionNum:" + partitionNum)

// 根据每个house目录生成DataFrame, 过程略

val hDF:DataFrame = null // 调用生成DataFrame的逻辑，这里使用null占位

//最近1个小时的log

val curHourDF = hDF.filter(s"acctime>='$curHourTime'")

// 1个小时之前的log

val preHourDF = hDF.filter(s"acctime>'$beginTime' and acctime

// 1个小时之前的数据， 数据量小， 分区数设置为1/3分区大小

val preHourPartNum = if(partitionNum/3 == 0) 1 else partitionNum/3

val newDF = curHourDF.coalesce(partitionNum).unionAll(preHourDF.coalesce(preHourPartNum))

if(resultDF != null){

resultDF = resultDF.unionAll(newDF)

}else{

resultDF = newDF

}

})

// 将DataFrame保存到HDFS(代码略)

""

}catch {

case e:Exception =>{

e.printStackTrace()

e.getMessage

}

}

}

def renameHDFSDir(fileSystem: FileSystem, srcLocation: String, destLocation: String): Boolean = {

val srcPath = new Path(srcLocation)

val destPath = new Path(destLocation)

val isRename = fileSystem.rename(srcPath, destPath)

isRename

}

}