GeoSpark实现热点分析案例

import com.vividsolutions.jts.geom.{Envelope, Geometry}
import com.vividsolutions.jts.io.WKTReader
import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.rdd.RDD
import org.apache.spark.serializer.KryoSerializer
import org.apache.spark.storage.StorageLevel
import org.apache.spark.{SparkConf, SparkContext}
import com.geoway.gspark.index.GTileRddMeta
import com.geoway.gspark.io.output.ShapefileOutput

import scala.collection.mutable.ArrayBuffer

/**
  * Created by wangchaojie on 2021/9/20.
  */
object GHotAnalysis extends Logging {

  def main(args: Array[String]): Unit = {

    val conf = new GSparkConf(args)
    if (!checkAppArgs(conf))
      sys.exit(-1)
    conf.setAppName("GHotAnalysis")
      .setMaster("local[4]") //DEBUG only

    val withKryo = conf.getBoolValue(COMMAND.kryo, true)
    if (withKryo) {
      logInfo("Kryo Serialization is ON.")
      conf.set("spark.serializer", classOf[KryoSerializer].getName)
      conf.set("spark.kryo.registrator", classOf[GoverlayKryoRegistrator].getName);
    }
    val sc = new SparkContext(conf)
    //sc.hadoopConfiguration.set("fs.defaultFS", GSparkConf.FS_DEFAULT)
    sc.hadoopConfiguration.set("fs.defaultFS", "hdfs://127.0.0.1:9000")
    logInfo("===== start tileUnion ====")
    //doTiledIntersect(sc)
    doAnalysis(sc)
    logInfo("===== finished tileUnion ====")
  }

  def doAnalysis(sc: SparkContext) = {
    val conf = sc.getConf
    val ileft = conf.get(COMMAND.getName(COMMAND.ileft))
    val iright = conf.get(COMMAND.getName(COMMAND.iright))
    val otype = conf.get(COMMAND.getName(COMMAND.otype)).toUpperCase()
    val fmtType = conf.get(COMMAND.getName(COMMAND.geoformat), "WKB")
    var iType = conf.get(COMMAND.getName(COMMAND.itype)).toUpperCase()
    val isDBType: Boolean = (iType != "HDFS")
    val tilelevel = conf.get(COMMAND.getName(COMMAND.tilelevel), "10")
    val oShpPath = conf.get(COMMAND.getName(COMMAND.oshapefilepath), "C:\\shpfile.shp")


    var resultRDD: RDD[GShape] = null

    var a = System.currentTimeMillis()
    val lfDbcfg = parseDBConfig(conf, true, true)
    var lfTileRdd = new GTileFormatRDD(sc, lfDbcfg, tilelevel.toInt)
    lfTileRdd.cache()
   println("个数"+ lfTileRdd.count())
    val lfMeta: GTileRddMeta = GTileRddMeta(lfTileRdd)
    val leftIndexRdd = new GTileRDD(lfTileRdd, lfMeta) //转为索引瓦片RDD
    leftIndexRdd.cache() //.persist(StorageLevel.MEMORY_ONLY_SER)//
    var xgrid: GIndexedGrid= null
    val unionRdd = lfTileRdd.zipPartitions(leftIndexRdd, true)(doPar)
    unionRdd.count()
  /*  resultRDD = lfTileRdd.map(p => hotAnalysis(p,xgrid))
    resultRDD.count()*/
    saveShapeFile(resultRDD.collect().iterator, oShpPath)
  }
  def doPar(shpIt: Iterator[GShape], idxIt: Iterator[GIndexedGrid]): Iterator[GShape] = {
    val arrRes = new ArrayBuffer[GShape]
    var xgrid: GIndexedGrid = idxIt.next()
    shpIt.foreach(shp => {
      arrRes+=hotAnalysis(shp,xgrid)
    })
    arrRes.iterator
  }
  def hotAnalysis(shape: GShape, xgrid: GIndexedGrid): GShape = {

    var bufferGeo = shape.geometry.buffer(0.1)
    val candidates = xgrid.query(bufferGeo.getEnvelopeInternal)
    if (candidates == null || candidates.length < 1) {

    }
    val candidates1 =new ArrayBuffer[GShape]()
        candidates.foreach(elem => {
      try {
        if (elem.intersects(shape)) {

            candidates1+=elem

        }
      } catch {
        case ex: Exception => {
          //GUtil.logStacktrace(ex, _log)
          println(s"热点分析异常：${ex.toString}")
        }
      }
    })


    val n = candidates1.length

    var mean = GetMean(candidates1)
    var sValue = GetS(mean,candidates1)
    var giValue = GetGiValue(mean,sValue,candidates1)

    var shapeResuult: GShape = new GShape(shape.oid, shape.geometry, Map("value"->giValue.toString))

    shapeResuult
  }

  def GetMean(candidates: ArrayBuffer[GShape]): Double = {
    var sumValue = 0.0
    candidates.foreach(elem => {
      sumValue += elem.attr.get("value").toString().toDouble
    })
    sumValue / candidates.length
  }

  def GetS(mean: Double, candidates: ArrayBuffer[GShape]): Double = {
    var sValue: Double = 0.0
    var quaSum = 0.0
    candidates.foreach(elem => {
      quaSum += elem.attr.get("value").toString().toDouble * elem.attr.get("value").toString().toDouble
    })
    sValue = Math.sqrt((quaSum / candidates.length) - mean * mean)
    sValue
  }

  def GetGiValue(mean: Double, sValue: Double, candidates: ArrayBuffer[GShape]): Double = {
    var giValue: Double = 0.0
    var quaSum = 0.0
    var upper1 = 0.0
    var upper2 = 0.0
    var upper = 0.0
    var low1 = 0.0
    var low2 = 0.0
    var low = 0.0
    var n = candidates.length
    candidates.foreach(elem => {
      upper1 += elem.attr.get("value").toString().toDouble * 1
      upper2 += mean * 1
      low1 += 1 * 1
      low2 += 1
    })
    upper = upper1 - upper2
    low = Math.sqrt((low1 * n - low2 * low2) / (n - 1)) * sValue
    giValue=upper/low
    giValue
  }
}