Spark MLlib特征处理：Normalizer 正则化---原理及实战

 

 

实战
import org.apache.spark.ml.feature.Normalizer
import org.apache.spark.sql.SQLContext
import org.apache.spark.{SparkContext, SparkConf}

object NormalizerExample {
  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("NormalizerExample").setMaster("local[6]")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)

    // LibSVMFile的格式：标签 索引：值
    // 0 2:51 3:253 5:253
    // 1 2:124 3:253 4:255
    // 1 2:145 3:253 5:211
    val dataFrame = sqlContext.read.format("libsvm").load("data/libsvm.txt")

    // L1正则化
    val normalizer = new Normalizer().setInputCol("features").setOutputCol("normFeatures")
      // 设置 L1正则化
      .setP(1.0)
    // 正则化转换
    val l1NormData = normalizer.transform(dataFrame)
    //l1NormData.foreach(println)
    // 结果：
    // [1.0,(5,[1,2,4],[145.0,253.0,211.0]),(5,[1,2,4],[0.23809523809523808,0.4154351395730706,0.3464696223316913])]
    // [0.0,(5,[1,2,4],[51.0,253.0,253.0]),(5,[1,2,4],[0.09156193895870736,0.4542190305206463,0.4542190305206463])]
    // [1.0,(5,[1,2,3],[124.0,253.0,255.0]),(5,[1,2,3],[0.1962025316455696,0.40031645569620256,0.40348101265822783])]
    // 结果分析：
    // 如：向量(5,[1,2,4],[145.0,253.0,211.0])
    // p-Norm=145.0+253.0+211.0=609.0   p=1
    // L1正则化：(5,[1,2,4],[145.0/609.0,253.0/609.0,211.0/609.0])
    // 正则化结果：(5,[1,2,4],[0.23809523809523808,0.4154351395730706,0.3464696223316913])]

    // L2正则化
     val l2InfNormData = normalizer.transform(dataFrame, normalizer.p -> 2)
    l2InfNormData.foreach(println)
    // 结果：
    // [0.0,(5,[1,2,4],[51.0,253.0,253.0]),(5,[1,2,4],[0.1411129915384288,0.7000311148867153,0.7000311148867153])]
    // [1.0,(5,[1,2,4],[145.0,253.0,211.0]),(5,[1,2,4],[0.40284772269065683,0.702899819591284,0.5862128930188178])]
    // [1.0,(5,[1,2,3],[124.0,253.0,255.0]),(5,[1,2,3],[0.3263044910373298,0.6657664212293906,0.6710293968912829])]
    // 结果分析：
    // 如：向量(5,[1,2,4],[145.0,253.0,211.0])
    // p-Norm=math.sqrt(145*145+253*253+211*211)=359.9374945737107 p=2
    // L2正则化：(5,[1,2,4],[145.0/p-Norm,253.0/p-Norm,211.0/p-Norm])
    // 正则化结果：(5,[1,2,4],[0.40284772269065683,0.702899819591284,0.5862128930188178])]
    sc.stop()

  }

}
--------------------- 
作者：wangpei1949 
来源：CSDN 
原文：https://blog.csdn.net/wangpei1949/article/details/53150089?utm_source=copy 
版权声明：本文为博主原创文章，转载请附上博文链接！