sparkMLlib逻辑线性回归demo

这里写一个逻辑线性回归demo 

import org.apache.log4j.{Level, Logger}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.mllib.regression.{IsotonicRegression, IsotonicRegressionModel, LabeledPoint}

object IsotonicRegression {
  def main(args: Array[String]) {
    // 设置运行环境
    val conf = new SparkConf().setAppName("Istonic Regression Test")
      .setMaster("spark://master:7077").setJars(Seq("E:\\Intellij\\Projects\\MachineLearning\\MachineLearning.jar"))
    val sc = new SparkContext(conf)
//设置控制台的输出级别
    Logger.getRootLogger.setLevel(Level.WARN)

    // 读取样本数据并解析
    val dataRDD = sc.textFile("hdfs://master:9000/ml/data/sample_isotonic_regression_data.txt")
    val parsedDataRDD = dataRDD.map { line =>
      val parts = line.split(',').map(_.toDouble)
      (parts(0), parts(1), 1.0)
    }

    // 样本数据划分,训练样本占0.7,测试样本占0.3
    val dataParts = parsedDataRDD.randomSplit(Array(0.7, 0.3), seed = 25L)
    val trainRDD = dataParts(0)
    val testRDD = dataParts(1)

    // 建立保序回归模型并训练
    val model = new IsotonicRegression().setIsotonic(true).run(trainRDD)
// 计算误差
    val prediction = testRDD.map { line =>
      val predicted = model.predict(line._2)
      (predicted, line._2, line._1)
    }
    //转成集合，便于输出打印
    val showPrediction = prediction.collect
    println
    println("Prediction" + "\t" + "Feature")
    for (i <- 0 to showPrediction.length - 1) {
      println(showPrediction(i)._1 + "\t" + showPrediction(i)._2)
    }
    val MSE = prediction.map { case (p, _, l1) => math.pow((p - l1), 2) }.mean()
    println("MSE = " + MSE)
  }
}