spark word2vec 源码_Spark文本特征提取（TF-IDF/Word2Vec/CountVectorizer）

importorg.apacheimportorg.apache.sparkimportorg.apache.spark.ml.feature._importorg.apache.spark.mllib.linalg._importorg.apache.spark.mllib.linalg.Vectorsimportorg.apache.spark.mllib.linalg.distributed.RowMatriximportorg.apache.spark.mllib.regression.LabeledPointimportorg.apache.spark.mllib.stat.{MultivariateStatisticalSummary, Statistics}importorg.apache.spark.rdd.RDDimportorg.apache.spark.sql.SQLContextimportorg.apache.spark.{SparkConf, SparkContext}importorg.apache.spark.mllib.linalg.{Matrices, Matrix}

object test42 {

def main(args: Array[String]): Unit={

val masterUrl= "local[2]"val appName= "tfidf_test"val sparkConf= newSparkConf().setMaster(masterUrl).setAppName(appName)

@transient val sc = newSparkContext(sparkConf)

val sqlContext= newSQLContext(sc)

sc.setLogLevel("ERROR")//Scala默认会导入scala.collection.immutable.Vector，//所以必须显式导入org.apache.spark.mllib.linalg.Vector才能使用MLlib才能使用MLlib提供的Vector。//密集向量

val dv:Vector = Vectors.dense(1.0,0.0,3.0)

println(dv)//稀疏向量,3表示此向量的长度，第一个Array(0,2)表示的索引，第二个Array(1.0, 3.0)与前面的Array(0,2)是相互对应的，表示第0个位置的值为1.0，第2个位置的值为3

val sv1:Vector=Vectors.sparse(3,Array(0,2),Array(1.0,3.0))

println(sv1)//稀疏向量, 3表示此向量的长度,Seq里面每一对都是(索引，值)的形式

val sv2:Vector=Vectors.sparse(3,Seq((0,1.0),(2,3.0)))

println(sv2)//标记点

val pos=LabeledPoint(1.0,Vectors.dense(1.0,0.0,3.0))

val neg=LabeledPoint(0.0,Vectors.sparse(3,Array(0,2),Array(1.0,3.0)))//创建矩阵,3行2列

val dm:Matrix=Matrices.dense(2,3,Array(1,0,2.0,3.0,4.0,5.0))

println("========dm========")

println(dm)

val v0= Vectors.dense(1.0, 0.0, 3.0)

val v1= Vectors.sparse(3, Array(1), Array(2.5))

val v2= Vectors.sparse(3, Seq((0, 1.5), (1, 1.8)))

val rows=sc.parallelize(Seq(v0, v1, v2))

println("=========rows=======")

println(rows.collect().toBuffer)

val mat: RowMatrix= newRowMatrix(rows)

val seriesX: RDD[Double]=sc.parallelize(List(1.0,2.0,3.0)) //a series

val seriesY: RDD[Double] = sc.parallelize(List(4.0,5.0,6.0)) //和seriesX必须有相同的分区和基数

val correlation:Double = Statistics.corr(seriesX, seriesY, "pearson")

val data: RDD[Vector]=rows//每个向量必须是行，不能是列

val correlMatrix: Matrix = Statistics.corr(data, "pearson")

println("========correlMatrix========")

println(correlMatrix)

val summary: MultivariateStatisticalSummary=Statistics.colStats(rows)

println("===================")

println(summary.mean)//每个列值组成的密集向量

println(summary.variance) //列向量方差

println(summary.numNonzeros) //每个列的非零值个数

/*** Word2Vec*/val documentDF=sqlContext.createDataFrame(Seq("Hi I heard about Spark".split(" "),"I wish Java could use case classes".split(" "),"Logistic regression models are neat".split(" ")

).map(Tuple1.apply)).toDF("text")//Learn a mapping from words to Vectors.

val word2Vec = newWord2Vec()

.setInputCol("text")

.setOutputCol("result")

.setVectorSize(3)

.setMinCount(0)

val model=word2Vec.fit(documentDF)

val result=model.transform(documentDF)

println("=======word2vec=========")

result.show(10,false)/*** Countvectorizer*/val df=sqlContext.createDataFrame(Seq(

(0, Array("a", "b", "c")),

(1, Array("a", "b", "b", "c", "a"))

)).toDF("id", "words")//fit a CountVectorizerModel from the corpus

val cvModel: CountVectorizerModel = newCountVectorizer()

.setInputCol("words")

.setOutputCol("features")

.setVocabSize(3)

.setMinDF(2)

.fit(df)//alternatively, define CountVectorizerModel with a-priori vocabulary

val cvm = new CountVectorizerModel(Array("a", "b", "c"))

.setInputCol("words")

.setOutputCol("features")

println("=======CountVectorizerModel=========")

cvModel.transform(df).show(10,false)/*** TF-IDF*/val sentenceData=sqlContext.createDataFrame(Seq(

(0, "Hi I heard about Spark"),

(0, "I wish Java could use case classes"),

(1, "Logistic regression models are neat")

)).toDF("label", "sentence")

val tokenizer= new Tokenizer().setInputCol("sentence").setOutputCol("words")

val wordsData=tokenizer.transform(sentenceData)

val hashingTF= newHashingTF()

.setInputCol("words").setOutputCol("rawFeatures").setNumFeatures(20)

val featurizedData=hashingTF.transform(wordsData)//CountVectorizer也可获取词频向量

val idf= new IDF().setInputCol("rawFeatures").setOutputCol("features")

val idfModel=idf.fit(featurizedData)

val rescaledData=idfModel.transform(featurizedData)

rescaledData.show(10,false)

}

}