源代码是/home/fhqplzj/.m2/repository/org/apache/spark/spark-mllib_2.11/2.0.0/spark-mllib_2.11-2.0.0-sources.jar!/org/apache/spark/mllib/util/DataValidators.scala
主要作用是标签检查:
package clustering.garbage
import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.rdd.RDD
import org.slf4j.LoggerFactory
/**
* Created by fhqplzj on 16-12-10 at 上午10:26.
*/
object DataValidators {
val LOG = LoggerFactory.getLogger("DataValidators")
/**
* 标签既不为0也不为1
*/
val binaryLabelValidator: RDD[LabeledPoint] => Boolean = {
data =>
val numInvalid = data.filter {
x =>
x.label != 0.0 && x.label != 1.0
}.count()
if (numInvalid != 0) {
LOG.error(s"Classification labels should be 0 or 1. Found $numInvalid invalid labels")
}
numInvalid == 0
}
/**
* 标签不是整数或者标签不在