文本向量化 java,文本的词条化和向量化

/***

* @author YangXin

* @info 此代码展示了如何对文本中的所有单词进行编码， 然后产生每个单词编码的线性权重之和，

* 从而将文本编码为向量。这是用StaticWordValueEncoder实现的，并且还要有办法将文本分解

* 或分析称单词。Mahout提供了编辑器，Lucene提供了分析器。

*/

package unitFourteen;

import java.io.IOException;

import java.io.StringReader;

import org.apache.commons.collections.bag.SynchronizedSortedBag;

import org.apache.lucene.analysis.Analyzer;

import org.apache.lucene.analysis.TokenStream;

import org.apache.lucene.analysis.standard.StandardAnalyzer;

import org.apache.lucene.analysis.tokenattributes.TermAttribute;

import org.apache.lucene.util.Version;

import org.apache.mahout.math.RandomAccessSparseVector;

import org.apache.mahout.math.SequentialAccessSparseVector;

import org.apache.mahout.math.Vector;

import org.apache.mahout.vectorizer.encoders.FeatureVectorEncoder;

import org.apache.mahout.vectorizer.encoders.StaticWordValueEncoder;

public class TokenizingAndVectorizingText {

public static void main(String[] args) throws IOException {

FeatureVectorEncoder encoder = new StaticWordValueEncoder("text");

Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_31);

StringReader in = new StringReader("text to magically vectorize");

TokenStream ts = analyzer.tokenStream("body", in);

TermAttribute termAtt = ts.addAttribute(TermAttribute.class);

Vector v1 = new RandomAccessSparseVector(100);

while (ts.incrementToken()) {

char[] termBuffer = termAtt.termBuffer();

int termLen = termAtt.termLength();

String w = new String(termBuffer, 0, termLen);

encoder.addToVector(w, 1, v1);

}

System.out.printf("%s

", new SequentialAccessSparseVector(v1));

}

}