Spark常用封装类

创建Sparksession对象

// 创建sparksession对象

import org.apache.spark.sql.SparkSession
import org.apache.spark.{SparkConf, SparkContext}

object SparkUtils {
    /**
     * 获取到本地的SparkContext对象
     */
    def getLocalSparkContext() :SparkContext = getLocalSparkContext("default_app")

    /**
     * 获取到本地的SparkContext对象
     */
    def getLocalSparkContext(appName: String): SparkContext = getSparkContext("local[*]", appName)

    /**
     * 获取到SparkContext对象
     */
    def getSparkContext(master: String, appName: String): SparkContext = new SparkContext(master, appName, new SparkConf())

    def getLocalSparkSession(): SparkSession = getLocalSparkSession("default_app", false)

    def getLocalSparkSession(appName: String, isSupportedHive: Boolean = false): SparkSession = getSparkSession("local[*]", appName, isSupportedHive)

    def getSparkSession(master: String, appName: String, isSupportedHive: Boolean = false): SparkSession = {
        if (isSupportedHive) SparkSession.builder().appName(appName).master(master).enableHiveSupport().getOrCreate()
        else SparkSession.builder().appName(appName).master(master).getOrCreate()
    }

    /**
     * 释放资源
     */
    def close(sc: SparkContext): Unit = if (sc != null && !sc.isStopped) sc.stop()

    def close(spark: SparkSession): Unit = if (spark != null && !spark.sparkContext.isStopped) spark.stop()
}

创建sparkContext对象

object SparkUtils {

    def getLocalSparkContext():SparkContext = getLocalSparkContext("default_app")

    def getLocalSparkContext(appName:String):SparkContext = getSparkContext("local[*]",appName)

    def getSparkContext(master:String, appName:String):SparkContext = new SparkContext("local[*]",appName, new SparkConf())

    def close(sc:SparkContext):Unit = {
        if(sc != null && !sc.isStopped) sc.stop()
    }
}

去除日志

//去除日志

object LoggerTrait {
    import org.apache.log4j.{Level, Logger}

    trait LoggerTrait {
        Logger.getLogger("org").setLevel(Level.WARN)
    }
}

ip转地址封装类

// ip转地址封装类

object IPUtils {
    def binarySearch(ipNum:Long, ipArr:Array[(String,Long,Long)]):Int = {
        var start = 0
        var end = ipArr.length - 1

        while(start <= end) {
            //获取中间的索引位置
            val mid = (start + end) / 2

            val startIP:Long = ipArr(mid)._2
            val endIP:Long = ipArr(mid)._3
            //判断
            if(ipNum >= startIP && ipNum <= endIP) return mid
            else if(ipNum < startIP) end = mid - 1
            else start = mid + 1
        }
        return -1
     }

    //将ip的字符转换为一个地址
    def ip2Address(ipStr:String, ipArr:Array[(String,Long,Long)]):String = {
        //将目标ip转化为数字
        val ipNum:Long = ip2Long(ipStr)
        //二分查找法
        val index = binarySearch(ipNum,ipArr)
        //获取地址
        if(index != -1)return ipArr(index)._1
        else return ""
    }


    //将ip字符串形式转换为ip数字的表示形式
    def ip2Long(ip: String):Long = {
        //切割字符串的ip
        val fields: Array[String] = ip.split("\\.")
        // 遍历数组，就能获取到每个ip的数字
        var ipNum = 0L
        fields.foreach(field => {
            ipNum = field.toLong | ipNum << 8
        })
        ipNum
    }
}

日期转换类封装

// 日期转换类封装

import java.text.SimpleDateFormat
import java.util.Date

object DateUtils {

    private val DATE_FORMAT:String = "yyyyMMddHHmmss"


    def dateStr2Date(dateStr: String, DATE_FORMAT: String):Date = {
        //获取日期格式化对象
        val format = new SimpleDateFormat(DATE_FORMAT)

        format.parse(dateStr)
    }

    def dataStr2Timestamep(dateStr:String):Long = {
        //将date的字符串转换为Date类型
        val date:Date = dateStr2Date(dateStr,DATE_FORMAT)
        //获取到date对应的时间戳
        date.getTime
    }
}