ETL常用工具类

字符相关

public class CharsetUtils {
    private enum Charset {
        /**
         * 7位ASCII字符，也叫作ISO646-US、Unicode字符集的基本拉丁块
         */
        US_ASCII("US-ASCII", "位ASCII字符，也叫作ISO646-US、Unicode字符集的基本拉丁块 "),
        ISO_8859_1("ISO-8859-1", "ISO 拉丁字母表 No.1，也叫作 ISO-LATIN-1"),
        GBK("GBK", "中文超大字符集"),
        UTF_8("UTF-8", "8 位 UCS 转换格式"),
        UTF_16BE("UTF-16BE", "16 位 UCS 转换格式，Big Endian（最低地址存放高位字节）字节顺序"),
        UTF_16LE("UTF_16LE", "16 位 UCS 转换格式，Big Endian（最低地址存放高位字节）字节顺序"),
        UTF_16("UTF_16", "16 位 UCS 转换格式，字节顺序由可选的字节顺序标记来标识");
        private String encode;
        private String desc;

        public String getEncode() {
            return encode;
        }

        public void setEncode(String encode) {
            this.encode = encode;
        }

        public String getDesc() {
            return desc;
        }

        public void setDesc(String desc) {
            this.desc = desc;
        }

        private Charset(String encode, String desc) {
            this.encode = encode;
            this.desc = desc;

        }


    }



    /**
     * 获取传入字符串的编码格式
     *
     * @param str
     * @return
     */
    public static String getEncode(String str) throws UnsupportedEncodingException {
        if (!StringUtils.isEmpty(str)) {
            for (Charset charset : Charset.values()) {
                String tryStr = new String(str.getBytes(charset.getEncode()), charset.getEncode());
                if (str.equals(tryStr))
                    return charset.getEncode();
            }
        }
        throw new UnsupportedEncodingException("编码库中不存在");
    }

    /**
     * 字符串编码转换的实现方法
     *
     * @param str        待转换编码的字符串
     * @param newCharset 目标编码
     * @return
     * @throws UnsupportedEncodingException
     */
    public static String changeCharset(String str, String newCharset)
            throws UnsupportedEncodingException {
        if (str != null) {
            //获取到原字符编码
            String charsetName = getEncode(str);
            //用默认字符编码解码字符串。
            byte[] bs = str.getBytes(charsetName);
            //用新的字符编码生成字符串
            return new String(bs, newCharset);
        }
        return null;
    }

    public static String changeCharset(String str, String oldCharset, String newCharset)
            throws UnsupportedEncodingException {
        if (str != null) {
            //获取到原字符编码
            String charsetName = oldCharset;
            //用默认字符编码解码字符串。
            byte[] bs = str.getBytes(charsetName);
            //用新的字符编码生成字符串
            return new String(bs, newCharset);
        }
        return null;
    }


    /**
     * 将字符编码转换成US-ASCII码
     */
    public static String toASCII(String str) throws UnsupportedEncodingException {
        return changeCharset(str, Charset.US_ASCII.getEncode());
    }

    /**
     * 将字符编码转换成ISO-8859-1码
     */
    public static String toISO_8859_1(String str) throws UnsupportedEncodingException {
        return changeCharset(str, Charset.ISO_8859_1.getEncode());
    }

    /**
     * 将字符编码转换成UTF-8码
     */
    public static String toUTF_8(String str) throws UnsupportedEncodingException {
        return changeCharset(str, Charset.UTF_8.getEncode());
    }

    /**
     * 将字符编码转换成UTF-16BE码
     */
    public static String toUTF_16BE(String str) throws UnsupportedEncodingException {
        return changeCharset(str, Charset.UTF_16BE.getEncode());
    }

    /**
     * 将字符编码转换成UTF-16LE码
     */
    public static String toUTF_16LE(String str) throws UnsupportedEncodingException {
        return changeCharset(str, Charset.UTF_16LE.getEncode());
    }

    /**
     * 将字符编码转换成UTF-16码
     */
    public static String toUTF_16(String str) throws UnsupportedEncodingException {
        return changeCharset(str, Charset.UTF_16.getEncode());
    }

    /**
     * 将字符编码转换成GBK码
     */
    public static String toGBK(String str) throws UnsupportedEncodingException {
        return changeCharset(str, Charset.GBK.getEncode());
    }

}

坐标相关

object CoordinateTransformUtil {
  val x_pi: Double = 3.14159265358979324 * 3000.0 / 180.0
  // π
  val pi = 3.1415926535897932384626
  // 长半轴
  val a = 6378245.0
  // 扁率
  val ee = 0.00669342162296594323

  /**
    * 百度坐标系(BD-09)转WGS坐标
    *
    * @param lng 百度坐标纬度
    * @param lat 百度坐标经度
    * @return WGS84坐标数组
    */
  def bd09towgs84(lng: Double, lat: Double): Array[Double] = {
    val gcj = bd09togcj02(lng, lat)
    val wgs84 = gcj02towgs84(gcj(0), gcj(1))
    wgs84
  }

  /**
    * WGS坐标转百度坐标系(BD-09)
    *
    * @param lng WGS84坐标系的经度
    * @param lat WGS84坐标系的纬度
    * @return 百度坐标数组
    */
  def wgs84tobd09(lng: Double, lat: Double): Array[Double] = {
    val gcj = wgs84togcj02(lng, lat)
    val bd09 = gcj02tobd09(gcj(0), gcj(1))
    bd09
  }

  /**
    * 火星坐标系(GCJ-02)转百度坐标系(BD-09)
    * <p>
    * 谷歌、高德——>百度
    *
    * @param lng 火星坐标经度
    * @param lat 火星坐标纬度
    * @return 百度坐标数组
    */
  def gcj02tobd09(lng: Double, lat: Double): Array[Double] = {
    val z = Math.sqrt(lng * lng + lat * lat) + 0.00002 * Math.sin(lat * x_pi)
    val theta = Math.atan2(lat, lng) + 0.000003 * Math.cos(lng * x_pi)
    val bd_lng = z * Math.cos(theta) + 0.0065
    val bd_lat = z * Math.sin(theta) + 0.006
    Array[Double](bd_lng, bd_lat)
  }

  /**
    * 百度坐标系(BD-09)转火星坐标系(GCJ-02)
    * <p>
    * 百度——>谷歌、高德
    *
    * @param bd_lon 百度坐标纬度
    * @param bd_lat 百度坐标经度
    * @return 火星坐标数组
    */
  def bd09togcj02(bd_lon: Double, bd_lat: Double): Array[Double] = {
    val x = bd_lon - 0.0065
    val y = bd_lat - 0.006
    val z = Math.sqrt(x * x + y * y) - 0.00002 * Math.sin(y * x_pi)
    val theta = Math.atan2(y, x) - 0.000003 * Math.cos(x * x_pi)
    val gg_lng = z * Math.cos(theta)
    val gg_lat = z * Math.sin(theta)
    Array[Double](gg_lng, gg_lat)
  }

  /**
    * WGS84转GCJ02(火星坐标系)
    *
    * @param lng WGS84坐标系的经度
    * @param lat WGS84坐标系的纬度
    * @return 火星坐标数组
    */
  def wgs84togcj02(lng: Double, lat: Double): Array[Double] = {
    if (out_of_china(lng, lat)) return Array[Double](lng, lat)
    var dlat = transformlat(lng - 105.0, lat - 35.0)
    var dlng = transformlng(lng - 105.0, lat - 35.0)
    val radlat = lat / 180.0 * pi
    var magic = Math.sin(radlat)
    magic = 1 - ee * magic * magic
    val sqrtmagic = Math.sqrt(magic)
    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
    dlng = (dlng * 180.0) / (a / sqrtmagic * Math.cos(radlat) * pi)
    val mglat = lat + dlat
    val mglng = lng + dlng
    Array[Double](mglng, mglat)
  }

  /**
    * GCJ02(火星坐标系)转GPS84
    *
    * @param lng 火星坐标系的经度
    * @param lat 火星坐标系纬度
    * @return WGS84坐标数组
    */
  def gcj02towgs84(lng: Double, lat: Double): Array[Double] = {
    if (out_of_china(lng, lat)) return Array[Double](lng, lat)
    var dlat = transformlat(lng - 105.0, lat - 35.0)
    var dlng = transformlng(lng - 105.0, lat - 35.0)
    val radlat = lat / 180.0 * pi
    var magic = Math.sin(radlat)
    magic = 1 - ee * magic * magic
    val sqrtmagic = Math.sqrt(magic)
    dlat = (dlat * 180.0) / ((a * (1 - ee)) / (magic * sqrtmagic) * pi)
    dlng = (dlng * 180.0) / (a / sqrtmagic * Math.cos(radlat) * pi)
    val mglat = lat + dlat
    val mglng = lng + dlng
    Array[Double](lng * 2 - mglng, lat * 2 - mglat)
  }

  /**
    * 纬度转换
    *
    * @param lng
    * @param lat
    * @return
    */
  def transformlat(lng: Double, lat: Double): Double = {
    var ret = -100.0 + 2.0 * lng + 3.0 * lat + 0.2 * lat * lat + 0.1 * lng * lat + 0.2 * Math.sqrt(Math.abs(lng))
    ret += (20.0 * Math.sin(6.0 * lng * pi) + 20.0 * Math.sin(2.0 * lng * pi)) * 2.0 / 3.0
    ret += (20.0 * Math.sin(lat * pi) + 40.0 * Math.sin(lat / 3.0 * pi)) * 2.0 / 3.0
    ret += (160.0 * Math.sin(lat / 12.0 * pi) + 320 * Math.sin(lat * pi / 30.0)) * 2.0 / 3.0
    ret
  }

  /**
    * 经度转换
    *
    * @param lng
    * @param lat
    * @return
    */
  def transformlng(lng: Double, lat: Double): Double = {
    var ret = 300.0 + lng + 2.0 * lat + 0.1 * lng * lng + 0.1 * lng * lat + 0.1 * Math.sqrt(Math.abs(lng))
    ret += (20.0 * Math.sin(6.0 * lng * pi) + 20.0 * Math.sin(2.0 * lng * pi)) * 2.0 / 3.0
    ret += (20.0 * Math.sin(lng * pi) + 40.0 * Math.sin(lng / 3.0 * pi)) * 2.0 / 3.0
    ret += (150.0 * Math.sin(lng / 12.0 * pi) + 300.0 * Math.sin(lng / 30.0 * pi)) * 2.0 / 3.0
    ret
  }

  /**
    * 判断是否在国内，不在国内不做偏移
    *
    * @param lng
    * @param lat
    * @return
    */
  def out_of_china(lng: Double, lat: Double): Boolean = {
    if (lng < 72.004 || lng > 137.8347) return true
    else if (lat < 0.8293 || lat > 55.8271) return true
    false
  }
}

时间相关

object DateTimeUtils {
  private val ft = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss")

  private[bigdata] val millis = (s: String) => {
    DateTime.parse(s, ft).getMillis / 1000
  }

  private[bigdata] val inHour = (millis: Long, sh: Int, eh: Int) => {
    val date = new DateTime(millis)
    val hour = date.hourOfDay.get
    hour >= sh && hour < eh
  }

  //每天的毫秒数
  private val day = 1000 * 60 * 60 * 24
  //日期字符串的格式
  private val format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")

  /**
    * 返回前一天的 00:00:00 字符串格式
    *
    * @return
    */
  def getLastDayStart: String = format.format(getTodayStartMills - day)

  /**
    * 返回前一天的 23:59:59 字符串格式
    *
    * @return
    */
  def getLastDayEnd: String = format.format(getTodayStartMills - 1)

  /**
    * 返回当日的 00:00:00 字符串格式
    *
    * @return
    */
  def getTodayStart: String = {
    val zero = getTodayStartMills
    format.format(zero)
  }

  /**
    * 返回当日的 23:59:59 字符串格式
    *
    * @return
    */
  def getTodayEnd: String = format.format(getTodayStartMills + day - 1)

  /**
    * 返回当日的 00:00:00 毫秒格式
    *
    * @return
    */
  def getTodayStartMills: Long = {
    val current = System.currentTimeMillis
    val zero = ((current + TimeZone.getDefault.getRawOffset) / day * day) - TimeZone.getDefault.getRawOffset
    zero
  }

  /**
    * 返回前一天的 00:00:00 毫秒格式
    *
    * @return
    */
  def getLastDayStartMills: Long = getTodayStartMills - day

  def main(args: Array[String]): Unit = {
    val inputPath = "/data/origin/vehicle/gps/taxi/440300/jiaowei/2019/10/08"
    //    println(path.substring(path.lastIndexOf("/")-7))

    //    val inputPath="/data/origin/vehicle/gps/taxi/440300/jiaowei/2019/10/09"
    //    val date = DateTime.parse(inputPath.substring(inputPath.lastIndexOf("/") - 7), DateTimeFormat.forPattern("yyyy/MM/dd")).plusDays(1)
    //    println(date.toString("yyyyMMdd HH:mm:ss"))

    val today = DateTime.parse(inputPath.substring(inputPath.lastIndexOf("/") - 7), DateTimeFormat.forPattern("yyyy/MM/dd")).plusDays(1)

    val time_tmp = "2019-10-07 23:59:22"
    //loc_time	定位日期	整型	10	否		unix10位时间戳
    val dataTime = DateTime.parse(time_tmp, DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"))

    val isOk = dataTime.getMillis >= today.minusDays(1).getMillis && dataTime.getMillis < today.getMillis

    println(isOk)

  }
}

加密

object DesUtils {

  def bytes2hex(bytes: Array[Byte]): String = {
    val hex = new StringBuilder()
    for (i <- 0 to bytes.length - 1) {
      val b = bytes(i)
      var negative = false
      if (b < 0) {
        negative = true
      }
      val inte = Math.abs(b)
      val temp = Integer.toHexString(inte & 0xFF)
      if (temp.length() == 1) {
        hex.append("0")
      }
      // hex.append(temp.toLowerCase())
      hex.append(temp)
    }
    hex.toString
  }

  // 生成AES密鈅
  @throws(classOf[Exception])
  def genKeyAES(): String = {
    val pkey = "123"
    val kgen = KeyGenerator.getInstance("DES")
    kgen.init(56, new SecureRandom(pkey.getBytes))
    val secretKey = kgen.generateKey
    val enCodeFormat = secretKey.getEncoded
    val key = new SecretKeySpec(enCodeFormat, "DES")
    //    val cipher = Cipher.getInstance("DES") // 创建密码器


    //    val keyGen = KeyGenerator.getInstance("DES")
    //    keyGen.init(56)
    //    val key = keyGen.generateKey()
    val base64Str = Base64.encodeBase64String(key.getEncoded())
    base64Str
  }

  @throws(classOf[Exception])
  def loadKeyAES(base64Key: String): SecretKey = {
    val bytes = Base64.decodeBase64(base64Key)
    val key = new SecretKeySpec(bytes, "DES")
    return key
  }

  def encryt(content: String): Array[Byte] = {
    encrytAES(content.getBytes(), loadKeyAES(genKeyAES))
    //    bytes2hex(encrytAES(content.getBytes(), loadKeyAES(genKeyAES)))
  }

  def encryt2Str(content: String): String = {
    bytes2hex(encrytAES(content.getBytes(), loadKeyAES(genKeyAES)))
  }

  @throws(classOf[Exception])
  def encrytAES(source: Array[Byte], key: SecretKey): Array[Byte] = {
    val cipher = Cipher.getInstance("DES")
    cipher.init(Cipher.ENCRYPT_MODE, key)
    cipher.doFinal(source)
  }

  def decry(source: Array[Byte]): String = {
    new String(decryptAES(source, loadKeyAES(genKeyAES)))
  }

  //  def decry2(source: String): String = {
  //    new String(decryptAES(hexStr2Byte(source), loadKeyAES(genKeyAES)))
  //  }

  @throws(classOf[Exception])
  def decryptAES(source: Array[Byte], key: SecretKey): Array[Byte] = {
    val cipher = Cipher.getInstance("DES")
    cipher.init(Cipher.DECRYPT_MODE, key)
    cipher.doFinal(source)
  }


  def main(args: Array[String]): Unit = {

    val content = "张三"
    println(decry(encryt(content)))


  }
}


object MD5Utils {

  def encode32(text: String): String = {
    val md = MessageDigest.getInstance("MD5")
    md.update(text.getBytes)
    val b = md.digest
    var i = 0
    val buf = new StringBuffer("")
    var offset = 0
    while ( {
      offset < b.length
    }) {
      i = b(offset)
      if (i < 0) i += 256
      if (i < 16) buf.append("0")
      buf.append(Integer.toHexString(i))

      {
        offset += 1;
        offset - 1
      }
    }
    buf.toString
  }

  def encode16(text: String): String = {
    encode32(text).substring(8, 24)
  }

  /** *
    * 任意文件转换成Md5
    * Can convert a text to MD5
    *
    * @param inputStream
    * @return md5
    */
  def encode(inputStream: InputStream): String = {
    var in = inputStream
    try {
      val digester = MessageDigest.getInstance("MD5")
      val bytes = new Array[Byte](8192)
      var byteCount = 0
      while ( {
        (byteCount = in.read(bytes));
        byteCount > 0
      }) digester.update(bytes, 0, byteCount)
      val digest = digester.digest
      val sb = new StringBuffer
      for (b <- digest) {
        val a = b & 0xff
        var hex = Integer.toHexString(a)
        if (hex.length == 1) hex = 0 + hex
        sb.append(hex)
      }
      return sb.toString
    } catch {
      case e: Exception =>
        e.printStackTrace()
    } finally if (in != null) {
      try
        in.close
      catch {
        case e: IOException =>
          e.printStackTrace()
      }
      in = null
    }
    null
  }

}

GIS计算

object Spatial {
  private val EARTH_RADIUS = 6378.137 // 地球半径

  // 使用经纬度估算两点之间的距离(米)
  private[bigdata] val distance = (lon1: Float, lat1: Float, lon2: Float, lat2: Float) => {
    val (l1, l2) = (rad(lat1), rad(lat2))
    val a = l1 - l2
    val b = rad(lon1) - rad(lon2)

    val d = EARTH_RADIUS * 2 * math.asin(math.sqrt(math.pow(math.sin(a / 2), 2) +
      math.cos(l1) * math.cos(l2) * math.pow(math.sin(b / 2), 2)))
    math.round(d * 10000) / 10
  }

  // 弧度到角度(PI=180°)
  private def rad(d: Float): Float = d * math.Pi.toFloat / 180
}

其他

object StdUtils {


  def isCnLngLat(lng: Double, lat: Double): Boolean = {

    //纬度3.86~53.55，经度73.66~135.05

    lng >= 73.66 && lng <= 135.05 && lat >= 3.86 && lat <= 53.55

  }

  def fillTime(date: String, pattern: String = "yyyy-MM-dd HH:mm:ss"): String = {

    if (StdUtils.isTime(date, Array(pattern))) {
      return date
    }

    var time1 = date

    if (date.length > 19) {
      time1 = date.substring(0, 19)
      return time1
    }

    val time2 = "1971-01-01 00:00:00"

    time1 + time2.substring(time1.length)

  }

  def main(args: Array[String]): Unit = {
    println(fillTime("2018-10-01 00:02:35.000"))
  }


  def isMonth(str: String): Boolean = {

    val parsePatterns = Array("yyyy/MM", "yyyy-MM", "yyyyMM")

    return isTime(str, parsePatterns)
  }


  //  val parsePatterns = Array("yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy/MM/dd", "yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyyMMdd")

  def isDate(str: String): Boolean = {

    val parsePatterns = Array("yyyy/MM/dd", "yyyy-MM-dd", "yyyyMMdd")

    return isTime(str, parsePatterns)

  }

  def isHour(str: String): Boolean = {

    val parsePatterns = Array("yyyy/MM/dd/HH", "yyyy-MM-dd HH", "yyyyMMdd HH")

    return isTime(str, parsePatterns)
  }

  //  def main(args: Array[String]): Unit = {
  //    val res = isTime("20571501040057", Array("yyyyMMddHHmmss"))
  //    println(res)
  //  }


  def isTime(str: String, parsePatterns: Array[String]): Boolean = {
    try {
      val date = DateUtils.parseDate(str, null, parsePatterns: _*)
      if (date.getTime > DateTime.now().getMillis) {
        return false;
      }
    } catch {
      case e: Exception => return false
    }

    return true;
  }


  def replaceBlank(str: String) = {
    str.replaceAll(" ", "")
  }

  /**
    * 临时替代数据年份
    *
    * @param date
    * @param year
    * @return
    */
  def replaceYear(date: String, year: String = "2017"): String = {
    if (!date.startsWith(year)) {
      year + date.substring(4)
    } else {
      date
    }
  }

  /**
    * 同济需求
    *
    * @param date
    * @param yearMonth
    * @return
    */
  def replaceYearMonth(date: String, yearMonth: String = "201710"): String = {
    if (date.startsWith("20190312")) {
      return "20171023"
    }
    if (!date.startsWith(yearMonth)) {
      yearMonth + date.substring(6)
    } else {
      date
    }
  }


  //  def main(args: Array[String]): Unit = {
  //   val arr = "2939131247,19980077,-15093343,201,2017-01-01 15:34:23,1,0,0,1,0,0,0,0,0,0,0,0,131856,131856,131856,131856,16.987055,16.987055,16.987055,16.987055,866,866,866,866,130896,130896,130896,130896,4.746764,4.746764,4.746764,4.746764,0,0,0,0,0,0,1".split(",")
  //
  //    println(arr)
  //    println(arr.length)
  //  }

  /**
    * 字符串前补0
    *
    * @param fillsize
    * @param data
    * @return
    */
  def fillZero(fillsize: Int, data: String): String = {


    val prefix = new StringBuilder

    Range(0, fillsize - data.trim.length).foreach(x => {
      prefix.append("0")
    })

    prefix.append(data.trim).toString()
  }


  /**
    * 判断全不为空
    *
    * @param arr
    * @return
    */
  def allNotEmpty(arr: Array[String]): Boolean = {
    var res = true
    for (e <- arr if res) {
      res = StringUtils.isNotEmpty(e)
    }
    res
  }


}