python3spark文本分类_[案例分析]基于Spark ML 的文本分类

清单 1. 示例程序源代码

import com.ibm.spark.exercise.util.LogUtils

import org.apache.spark.ml.Pipeline

import org.apache.spark.ml.classification.MultilayerPerceptronClassifier

import org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator

import org.apache.spark.ml.feature.{IndexToString, StringIndexer, Word2Vec}

import org.apache.spark.sql.SQLContext

import org.apache.spark.{SparkContext, SparkConf}

object SMSClassifier {

final val VECTOR_SIZE = 100

def main(args: Array[String]) {

if (args.length < 1) {

println("Usage:SMSClassifier SMSTextFile")

sys.exit(1)

}

LogUtils.setDefaultLogLevel()

val conf = new SparkConf().setAppName("SMS Message Classification (HAM or SPAM)")

val sc = new SparkContext(conf)

val sqlCtx = new SQLContext(sc)</