lucene中分词器的用法

package com.ljq.analyzer;

import java.io.StringReader;

import jeasy.analysis.MMAnalyzer;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.junit.Test;

public class AnalyzerTest {

String ensaText = " IndexWriter addDocument's a javadoc.txt " ;
String ensa2Text = " 我们是中国人 " ;
String zhcjkaText = " 小笑话_总统的房间 Room .txt " ;
String zhmnText = " 一位绅士到旅游胜地的一家饭店要开个房间 " ;

Analyzer ensa = new StandardAnalyzer(); // 单字分词
Analyzer ensa2 = new SimpleAnalyzer();
Analyzer zhcjka = new CJKAnalyzer(); // 二分法分词
Analyzer zhmn = new MMAnalyzer(); // 词库分词

@Test
public void test() throws Exception {
// 单字分词
/* (indexwriter,0,11,type=<ALPHANUM>)
(adddocument,12,25,type=<APOSTROPHE>)
(javadoc.txt,28,39,type=<HOST>) */
// analyze(ensa, ensaText);

// (我们是中国人,0,6)
// analyze(ensa2, ensa2Text);

// 二分法分词
/* (小笑,0,2,type=double)
(笑话,1,3,type=double)
(_,3,4,type=single)
(总统,4,6,type=double)
(统的,5,7,type=double)
(的房,6,8,type=double)
(房间,7,9,type=double)
(room,10,14,type=single)
(txt,16,19,type=single) */
// analyze(zhcjka, zhcjkaText);

// 词库分词
/* (一位,0,2)
(绅士,2,4)
(旅游胜地,5,9)
(一家,10,12)
(饭店,12,14)
(要,14,15)
(开个,15,17)
(房间,17,19) */
analyze(zhmn, zhmnText);
}

/**
* 分词
*
* @param analyzer
* 分词器
* @param text
* 数据源
* @throws Exception
*/
public void analyze(Analyzer analyzer, String text) throws Exception {
TokenStream tokenStream = analyzer.tokenStream( " content " ,
new StringReader(text));
for (Token token = new Token(); (token = tokenStream.next(token)) != null ;) {
System.out.println(token);
}
}
}