标签检查

源代码是/home/fhqplzj/.m2/repository/org/apache/spark/spark-mllib_2.11/2.0.0/spark-mllib_2.11-2.0.0-sources.jar!/org/apache/spark/mllib/util/DataValidators.scala

主要作用是标签检查：

package clustering.garbage

import org.apache.spark.mllib.regression.LabeledPoint
import org.apache.spark.rdd.RDD
import org.slf4j.LoggerFactory

/**
  * Created by fhqplzj on 16-12-10 at 上午10:26.
  */
object DataValidators {
  val LOG = LoggerFactory.getLogger("DataValidators")
  /**
    * 标签既不为０也不为１
    */
  val binaryLabelValidator: RDD[LabeledPoint] => Boolean = {
    data =>
      val numInvalid = data.filter {
        x =>
          x.label != 0.0 && x.label != 1.0
      }.count()
      if (numInvalid != 0) {
        LOG.error(s"Classification labels should be 0 or 1. Found $numInvalid invalid labels")
      }
      numInvalid == 0
  }

  /**
    * 标签不是整数或者标签不在