public class AnalyzerUtil {
public static void main(String[] args) {
StandardAnalyzer sa=new StandardAnalyzer(Version.LUCENE_45);
String str="I am come from jiangxi fengcheng tongtian . this is a dog";
TokenStream ts=null;
try {
ts=sa.tokenStream("content", new StringReader(str));
CharTermAttribute cta=ts.addAttribute(CharTermAttribute.class);
ts.reset();
while(ts.incrementToken())
{
//CharTermAttribute cta=ts.getAttribute(CharTermAttribute.class);
System.out.println(cta);
}
ts.end();
/*while(ts.incrementToken()){
System.out.println(cta);
}*/
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
if(ts!=null)
try {
ts.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}
输出:
i
am
come
from
beijing
dog
reset():This method is called by a consumer before it begins consumption using incrementToken().
如果没有加上ts.reset();会报空指针异常。如:
Exception in thread "main" java.lang.NullPointerException
at org.apache.lucene.analysis.standard.StandardTokenizerImpl.zzRefill(StandardTokenizerImpl.java:921)
at org.apache.lucene.analysis.standard.StandardTokenizerImpl.getNextToken(StandardTokenizerImpl.java:1128)
at org.apache.lucene.analysis.standard.StandardTokenizer.incrementToken(StandardTokenizer.java:173)
at org.apache.lucene.analysis.standard.StandardFilter.incrementToken(StandardFilter.java:49)
at org.apache.lucene.analysis.core.LowerCaseFilter.incrementToken(LowerCaseFilter.java:54)
at org.apache.lucene.analysis.util.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:82)
at com.lucene.analyzer.AnalyzerUtil.main(AnalyzerUtil.java:24)