Lucene3.5中有这四大类分词器:SimpleAnalyzer、StopAnalyzer、WhiltespaceAnalyzer、StandardAnalyzer。
分词流程:
Tokenizer:
TokenFilter:
TokenStream中保存的信息如下:
了解了分词流程和TokenStream之后,可以通过代码来查看分词信息:
package cn.liuys.lucene.util;
import java.io.IOException;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
public class AnalyzerUtil {
/**
* @param str
* @param a
* 显示分词详细信息
*/
public static void displayAllTokenInfo(String str,Analyzer a){
TokenStream stream = a.tokenStream("content", new StringReader(str));
//位置增量
PositionIncrementAttribute pia = stream.addAttribute(PositionIncrementAttribute.class);
//偏移量
OffsetAttribute oa = stream.addAttribute(OffsetAttribute.class);
//词汇单元
CharTermAttribute cta = stream.addAttribute(CharTermAttribute.class);
//类型
TypeAttribute ta = stream.addAttribute(TypeAttribute.class);
try {
for(;stream.incrementToken();){
System.out.print(pia.getPositionIncrement()+":");
System.out.print(cta+"["+oa.startOffset()+"-"+oa.endOffset()+"]--->"+ta.type()+"\n");
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
package cn.liuys.lucene.test;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.StopAnalyzer;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.util.Version;
import org.junit.Test;
import cn.liuys.lucene.util.AnalyzerUtil;
public class TestAnalyzer {
@Test
public void test03(){
Analyzer a1 = new StandardAnalyzer(Version.LUCENE_35);
Analyzer a2 = new StopAnalyzer(Version.LUCENE_35);
Analyzer a3 = new SimpleAnalyzer(Version.LUCENE_35);
Analyzer a4 = new WhitespaceAnalyzer(Version.LUCENE_35);
String str = "how are you thank you";
AnalyzerUtil.displayAllTokenInfo(str, a1);
System.out.println("-------------------------------------");
AnalyzerUtil.displayAllTokenInfo(str, a2);
System.out.println("-------------------------------------");
AnalyzerUtil.displayAllTokenInfo(str, a3);
System.out.println("-------------------------------------");
AnalyzerUtil.displayAllTokenInfo(str, a4);
}
}