spark高级数据分析---网络流量异常检测(升级实战)

版权声明:本文为博主原创文章,遵循 CC 4.0 BY-SA 版权协议,转载请附上原文出处链接和本声明。
本文链接:https://blog.csdn.net/sinat_31726559/article/details/52016652

在我的上一篇里我写的那个只是个人对KMeans聚类在这个项目中的一部分,今天花了很长时间写完和完整的运行测试完这个代码,篇幅很长,都是结合我前面写的加上自己完善的异常检测部分,废话不多说,直接代码实战:

package internet

import org.apache.spark.mllib.clustering.{KMeansModel, KMeans}
import org.apache.spark.mllib.linalg.{Vectors,Vector}
import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkContext, SparkConf}

/**
  * Created by 汪本成 on 2016/7/24.
  */
object CheckAll {

  def main(args: Array[String]) {
    //创建入口对象
    val conf = new SparkConf().setAppName("CheckAll").setMaster("local")
    val sc= new SparkContext(conf)
    val HDFS_DATA_PATH = "hdfs://node1:9000/user/spark/sparkLearning/cluster/kddcup.data"
    val rawData = sc.textFile(HDFS_DATA_PATH)

    /** 分类统计样本,降序排序 **/
//    clusteringTake1(rawData)
    /** 评价k**/
//    clusteringTake2(rawData)
//    clusteringTake3(rawData)
//    clusteringTake4(rawData)
//    clusteringTake5(rawData)
    /** R数据可视化 **/
    /** 异常检测 **/
    var beg = System.currentTimeMillis()
    anomalies(rawData)
    var end = System.currentTimeMillis()
    println("用时:" + (end - beg) / 1000 + "s")
  }



  //ClusteringTask1
  def clusteringTake1(rawData: RDD[String]) = {
    //分类统计样本个数,降序排序
    rawData.map(_.split(",").last).countByValue().toSeq.sortBy(_._2).reverse.foreach(println)


    val labelsAndData = rawData.map {
      line =>
        //csv格式的行拆分成列,创建一个buffer,是一个可变列表
        val buffer = line.split(",").toBuffer
        //删除下标从1开始的三个类别型列
        buffer.remove(1, 3)
        //删除下标最后的标号列
        val label = buffer.remove(buffer.length - 1)
        //保留其他值并将其转换成一个数值型(Double型对象)数组
        val vector = Vectors.dense(buffer.map(_.toDouble).toArray)
        //将数组和标号组成一个元祖
        (label, vector)
    }

    /**
      * 为啥要进行labelsAndData => data转化?
      * 1k均值在运行过程中只用到特征向量(即没有用到数据集的目标标号列)
      * 2、使data这个RDD只包含元祖的只包含元组的第二个元素
      * 3、实现2可以通过元组类型RDDvalues属性得到,在放入缓存中,减少落地
      */
    //提取出元组的特征向量
    val data = labelsAndData.values.cache()

    //实例化Kmeans类对象
    val kmeans = new KMeans()
    //建立KMeansModel
    val model = kmeans.run(data)
    //输出每个簇的质心
    model.clusterCenters.foreach(println)

    val clusterLabelCount = labelsAndData.map {
      case (label, datum) =>
        //预测样本datum的分类cluster
        val cluster = model.predict(datum)
        //返回类别-簇的元组
        (cluster, label)
    }.countByValue()

    //对簇-类别对分别进行计数,并以可读方式输出
    clusterLabelCount.toSeq.sorted.foreach {
      case ((cluster, label), count) =>
        println(f"$cluster%1s$label%18s$count%8s")
    }
    data.unpersist()
  }

  /**
    * 欧氏距离公式
    * a.toArray.zip(b.toArray)对应 "两个向量相应元素"
    * map(p => p._1 - p._2)对应 ""
    * map(d => d*d).sum对应 "平方和"
    * math.sqrt()对应 "平方根"
    * @param a
    * @param b
    * @return
    */
  def distance(a: Vector, b: Vector) =
    math.sqrt(a.toArray.zip(b.toArray).map(p => p._1 - p._2).map(d => d * d).sum)

  /**
    * 欧氏距离公式应用到model    * KMeansModel.predict方法中调用了KMeans对象的findCloest方法
    * @param datum
    * @param model
    * @return
    */
  def distToCenter(datum: Vector, model: KMeansModel) = {
    //预测样本datum的分类cluster
    val cluster = model.predict(datum)
    //计算质心
    val center = model.clusterCenters(cluster)
    //应用距离公式
    distance(center, datum)
  }

  /**
    * 平均质心距离
    * @param data
    * @param k
    * @return
    */
  def clusteringScore(data: RDD[Vector], k: Int): Double = {
    val kmeans = new KMeans()
    //设置k    kmeans.setK(k)
    //建立KMeansModel
    val model = kmeans.run(data)
    //计算kmodel平均质心距离,mean()是平均函数
    data.map(datum => distToCenter(datum, model)).mean()
  }

  /**
    * 平均质心距离优化
    * @param data
    * @param k
    * @param run   运行次数
    * @param epsilon  阈值
    * @return
    */
  def clusteringScore2(data: RDD[Vector], k: Int, run: Int, epsilon: Double): Double = {
    val kmeans = new KMeans()
    kmeans.setK(k)
    //设置k的运行次数
    kmeans.setRuns(run)
    //设置阈值
    kmeans.setEpsilon(epsilon)
    val model = kmeans.run(data)
    data.map(datum => distToCenter(datum, model)).mean()
  }

  //ClusteringTake2
  def clusteringTake2(rawData: RDD[String]): Unit ={
    val data = rawData.map {
      line =>
        val buffer = line.split(",").toBuffer
        buffer.remove(1, 3)
        buffer.remove(buffer.length - 1)
        Vectors.dense(buffer.map(_.toDouble).toArray)
    }.cache()

    val run = 10
    val epsilon = 1.0e-4
    //(5,30)区间内以5为等差数列数值不同k值对其评分
    (5 to 30 by 5).map(k => (k, clusteringScore(data, k))).foreach(println)
    //(20,120)区间内以10为等差数列数值不同k值对其评分
    (30 to 100 by 10).par.map(k => (k, clusteringScore2(data, k, run, epsilon))).foreach(println)

    data.unpersist()
  }


  /**
    * 加工出R可视化数据存入HDFS    * @param rawData
    * @param k
    * @param run
    * @param epsilon
    */
  def visualizationInR(rawData: RDD[String], k: Int, run: Int, epsilon: Double): Unit ={
    val data = rawData.map {
      line =>
        val buffer = line.split(",").toBuffer
        buffer.remove(1, 3)
        buffer.remove(buffer.length - 1)
        Vectors.dense(buffer.map(_.toDouble).toArray)
    }.cache()

    val kmeans = new KMeans()
    kmeans.setK(k)
    kmeans.setRuns(run)
    kmeans.setEpsilon(epsilon)
    val model = kmeans.run(data)

    val sample = data.map(
      datum =>
        model.predict(datum) + "," + datum.toArray.mkString(",")
    ).sample(false, 0.05)   //选择了5%
    sample.saveAsTextFile("hdfs://nodel:9000/user/spark/R/sample")
    data.unpersist()
  }

  /**
    *
    * @param data
    * @return
    */
  def buildNormalizationFunction(data: RDD[Vector]): (Vector => Vector) = {
    //将数组缓冲为Array
    val dataAsArray = data.map(_.toArray)
    //数据集第一个元素的长度
    val numCols = dataAsArray.first().length
    //返回数据集的元素个数
    val n = dataAsArray.count()
    //两个数组对应元素相加求和
    val sums = dataAsArray.reduce((a, b) => a.zip(b).map(t => t._1 + t._2))
    //RDD聚合后进行求平方和操作
    val sumSquares = dataAsArray.aggregate(new Array[Double](numCols))(
      (a, b) => a.zip(b).map(t => t._1 + t._2 * t._2),
      (a, b) => a.zip(b).map(t => t._1 + t._2)
    )

    /** zip函数将传进来的两个参数中相应位置上的元素组成一个pair数组。
      * 如果其中一个参数元素比较长,那么多余的参数会被删掉。
      * 个人理解就是让两个数组里面的元素一一对应进行某些操作
      */
    val stdevs = sumSquares.zip(sums).map {
      case (sumSq, sum) => math.sqrt(n * sumSq - sum * sum) / n
    }
    val means = sums.map(_ / n)

    (datum : Vector) => {
      val normalizedArray = (datum.toArray, means, stdevs).zipped.map(
        (value, mean, stdev) =>
          if(stdev <= 0) (value- mean) else (value - mean) /stdev
      )
      Vectors.dense(normalizedArray)
    }
  }


  //clusteringTask3
  def clusteringTake3(rawData: RDD[String]): Unit ={
    val data = rawData.map { line =>
      val buffer = line.split(',').toBuffer
      buffer.remove(1, 3)
      buffer.remove(buffer.length - 1)
      Vectors.dense(buffer.map(_.toDouble).toArray)
    }

    val run = 10
    val epsilon = 1.0e-4

    val normalizedData = data.map(buildNormalizationFunction(data)).cache()

    (60 to 120 by 10).par.map(
      k => (k, clusteringScore2(normalizedData, k, run, epsilon))
    ).toList.foreach(println)

    normalizedData.unpersist()
  }

  /**
    * 基于one-hot编码实现类别型变量替换逻辑
    * @param rawData
    * @return
    */
  def buildCategoricalAndLabelFunction(rawData: RDD[String]): (String => (String, Vector))  = {
    val splitData = rawData.map(_.split(","))
    //建立三个特征
    val protocols = splitData.map(_(1)).distinct().collect().zipWithIndex.toMap   //特征值是100
    val services = splitData.map(_(2)).distinct().collect().zipWithIndex.toMap    //特征值是010
    val tcpStates = splitData.map(_(3)).distinct().collect().zipWithIndex.toMap   //特征值是001
    //
    (line: String) => {
      val buffer = line.split(",").toBuffer
      val protocol = buffer.remove(1)
      val service = buffer.remove(1)
      val tcpState = buffer.remove(1)
      val label = buffer.remove(buffer.length - 1)
      val vector = buffer.map(_.toDouble)

      val newProtocolFeatures = new Array[Double](protocols.size)
      newProtocolFeatures(protocols(protocol)) = 1.0
      val newServiceFeatures = new Array[Double](services.size)
      newServiceFeatures(services(service)) = 1.0
      val newTcpStateFeatures = new Array[Double](tcpStates.size)
      newTcpStateFeatures(tcpStates(tcpState)) = 1.0

      vector.insertAll(1, newTcpStateFeatures)
      vector.insertAll(1, newServiceFeatures)
      vector.insertAll(1, newProtocolFeatures)

      (label, Vectors.dense(vector.toArray))
    }
  }


  //ClusteringTask4
  def clusteringTake4(rawData: RDD[String]): Unit ={
    val paraseFunction = buildCategoricalAndLabelFunction(rawData)
    val data = rawData.map(paraseFunction).values
    val normalizedData = data.map(buildNormalizationFunction(data)).cache()

    val run = 10
    val epsilon = 1.0e-4

    (80 to 160 by 10).map(
      k=> (k, clusteringScore2(normalizedData, k, run, epsilon))
    ).toList.foreach(println)

    normalizedData.unpersist()
  }


  //Clustering, Task5
  /**
    * 对各个簇的熵加权平均,将结果作为聚类得分
    * @param counts
    * @return
    */
  def entropy(counts: Iterable[Int]) = {
    val values = counts.filter(_ > 0)
    val n: Double = values.sum
    values.map {
      v =>
        val p = v / n
        -p * math.log(p)
    }.sum
  }

  /**
    * 计算熵的加权平均
    * @param normalizedLabelsAndData
    * @param k
    * @param run
    * @param epsilon
    * @return
    */
  def clusteringScore3(normalizedLabelsAndData: RDD[(String, Vector)], k: Int, run: Int, epsilon: Double) = {
    val kmeans = new KMeans()
    kmeans.setK(k)
    kmeans.setRuns(run)
    kmeans.setEpsilon(epsilon)

    //建立KMeansModel
    val model = kmeans.run(normalizedLabelsAndData.values)
    //对每个数据集预测簇类别
    val labelAndClusters = normalizedLabelsAndData.mapValues(model.predict)
    //RDD[(String, Vector)]  => RDD[(String, Vector)],swap Keys / Values,对换键和值
    val clustersAndLabels = labelAndClusters.map(_.swap)
    //按簇提取标号集合
    val labelsInCluster = clustersAndLabels.groupByKey().values
    //计算所有集合中有多少标签(label),即标号的出现次数
    val labelCounts = labelsInCluster.map(_.groupBy(l => l).map(_._2.size))
    //通过类别大小来反映平均信息量,即熵
    val n = normalizedLabelsAndData.count()
    //根据簇大小计算熵的加权平均
    labelCounts.map(m => m.sum * entropy(m)).sum() / n
  }


  def clusteringTake5(rawData: RDD[String]): Unit ={
    val parseFunction = buildCategoricalAndLabelFunction(rawData)
    val labelAndData = rawData.map(parseFunction)
    val normalizedLabelsAndData = labelAndData.mapValues(buildNormalizationFunction(labelAndData.values)).cache()

    val run = 10
    val epsilon = 1.0e-4

    (80 to 160 by 10).map(
      k => (k, clusteringScore3(normalizedLabelsAndData, k, run, epsilon))
    ).toList.foreach(println)

    normalizedLabelsAndData.unpersist()
  }


  //Detect anomalies(发现异常)
  def bulidAnomalyDetector(data: RDD[Vector], normalizeFunction: (Vector => Vector)): (Vector => Boolean) = {
    val normalizedData = data.map(normalizeFunction)
    normalizedData.cache()

    val kmeans = new KMeans()
    kmeans.setK(150)
    kmeans.setRuns(10)
    kmeans.setEpsilon(1.0e-6)
    val model = kmeans.run(normalizedData)

    normalizedData.unpersist()

    //度量新数据点到最近簇质心的距离
    val distances = normalizedData.map(datum => distToCenter(datum, model))
    //设置阀值为已知数据中离中心点最远的第100个点到中心的距离
    val threshold = distances.top(100).last

    //检测,若超过该阀值就为异常点
    (datum: Vector) => distToCenter(normalizeFunction(datum), model) > threshold
  }

  /**
    * 异常检测
    * @param rawData
    */
  def anomalies(rawData: RDD[String]) = {
    val parseFunction = buildCategoricalAndLabelFunction(rawData)
    val originalAndData = rawData.map(line => (line, parseFunction(line)._2))
    val data = originalAndData.values
    val normalizeFunction = buildNormalizationFunction(data)
    val anomalyDetector = bulidAnomalyDetector(data, normalizeFunction)
    val anomalies = originalAndData.filter {
      case (original, datum) => anomalyDetector(datum)
    }.keys
    //10个异常点打印出来
    anomalies.take(10).foreach(println)
  }
}
写的有点杂,但是全部自己封装好了。运行起来也没问题,仅供大家参考学习,多多关注下我写的注释就好。

累死我了,或许是我电脑不行缘故,计算这1G数据花了这么长时间,现在我把异常检测部分运行结果给大家看看好了

16/07/24 22:48:18 INFO Executor: Running task 0.0 in stage 65.0 (TID 385)
16/07/24 22:48:18 INFO HadoopRDD: Input split: hdfs://node1:9000/user/spark/sparkLearning/cluster/kddcup.data:0+134217728
16/07/24 22:48:30 INFO Executor: Finished task 0.0 in stage 65.0 (TID 385). 3611 bytes result sent to driver
16/07/24 22:48:30 INFO TaskSetManager: Finished task 0.0 in stage 65.0 (TID 385) in 11049 ms on localhost (1/1)
9,tcp,telnet,SF,307,2374,0,0,1,0,0,1,0,1,0,1,3,1,0,0,0,0,1,1,0.00,0.00,0.00,0.00,1.00,0.00,0.00,69,4,0.03,0.04,0.01,0.75,0.00,0.00,0.00,0.00,normal.
16/07/24 22:48:30 INFO TaskSchedulerImpl: Removed TaskSet 65.0, whose tasks have all completed, from pool
16/07/24 22:48:30 INFO DAGScheduler: ResultStage 65 (take at CheckAll.scala:413) finished in 11.049 s
16/07/24 22:48:30 INFO DAGScheduler: Job 41 finished: take at CheckAll.scala:413, took 11.052917 s
0,tcp,http,S1,299,26280,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,15,16,0.07,0.06,0.00,0.00,1.00,0.00,0.12,231,255,1.00,0.00,0.00,0.01,0.01,0.01,0.00,0.00,normal.
0,tcp,telnet,S1,2895,14208,0,0,0,0,0,1,0,0,0,0,13,0,0,0,0,0,1,1,1.00,1.00,0.00,0.00,1.00,0.00,0.00,21,2,0.10,0.10,0.05,0.00,0.05,0.50,0.00,0.00,normal.
23,tcp,telnet,SF,104,276,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,1,1,0.00,0.00,0.00,0.00,1.00,0.00,0.00,1,2,1.00,0.00,1.00,1.00,0.00,0.00,0.00,0.00,guess_passwd.
13,tcp,telnet,SF,246,11938,0,0,0,0,4,1,0,0,0,0,2,0,0,0,0,0,1,1,0.00,0.00,0.00,0.00,1.00,0.00,0.00,89,2,0.02,0.04,0.01,0.00,0.00,0.00,0.00,0.00,normal.
12249,tcp,telnet,SF,3043,44466,0,0,0,1,0,1,13,1,0,0,12,0,0,0,0,0,1,1,0.00,0.00,0.00,0.00,1.00,0.00,0.00,61,8,0.13,0.05,0.02,0.00,0.00,0.00,0.00,0.00,normal.
60,tcp,telnet,S3,125,179,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,1,1.00,1.00,0.00,0.00,1.00,0.00,0.00,1,1,1.00,0.00,1.00,0.00,1.00,1.00,0.00,0.00,guess_passwd.
60,tcp,telnet,S3,126,179,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,2,2,0.50,0.50,0.50,0.50,1.00,0.00,0.00,23,23,1.00,0.00,0.04,0.00,0.09,0.09,0.91,0.91,guess_passwd.
583,tcp,telnet,SF,848,25323,0,0,0,1,0,1,107,1,1,100,1,0,0,0,0,0,1,1,0.00,0.00,0.00,0.00,1.00,0.00,0.00,1,1,1.00,0.00,1.00,0.00,0.00,0.00,0.00,0.00,normal.
11447,tcp,telnet,SF,3131,45415,0,0,0,1,0,1,0,1,0,0,15,0,0,0,0,0,1,1,0.00,0.00,0.00,0.00,1.00,0.00,0.00,100,10,0.09,0.72,0.01,0.20,0.01,0.10,0.69,0.20,normal.
用时:4602s

16/07/24 22:48:30 INFO SparkContext: Invoking stop() from shutdown hook
16/07/24 22:48:30 INFO SparkUI: Stopped Spark web UI at http://192.168.1.102:4040
16/07/24 22:48:30 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
16/07/24 22:48:30 INFO MemoryStore: MemoryStore cleared
16/07/24 22:48:30 INFO BlockManager: BlockManager stopped
16/07/24 22:48:30 INFO BlockManagerMaster: BlockManagerMaster stopped
16/07/24 22:48:30 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
16/07/24 22:48:30 INFO SparkContext: Successfully stopped SparkContext
16/07/24 22:48:30 INFO ShutdownHookManager: Shutdown hook called
16/07/24 22:48:30 INFO ShutdownHookManager: Deleting directory C:\Users\Administrator\AppData\Local\Temp\spark-1ab0ec11-672d-4778-9ae8-2050f44a5f91
16/07/24 22:48:30 INFO RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon.
16/07/24 22:48:30 INFO RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports.

Process finished with exit code 0

运行结果的十条数据我已经标红,大家注意下,跑了我怕一个多小时时间,唉



展开阅读全文

没有更多推荐了,返回首页