1、我使用的jar为IKAnalyzer2012_u6.jar 请自行百度下载,并引用到项目中
2、pom.xml 添加引用
<dependency> <!-- 查询相关jar包 -->
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-queryparser</artifactId>
<version>6.0.0</version>
</dependency>
<dependency> <!-- lucene自带只能中文分词器jar包 -->
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-smartcn</artifactId>
<version>6.0.0</version>
</dependency>
<dependency> <!-- 测试用到的lucene工具包 -->
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>6.0.0</version>
</dependency>
<dependency> <!-- 测试用到的lucene核心包 -->
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>6.0.0</version>
</dependency>
3、创建IKAnalyzer6x.java 和IKTokenizer6x.java
IKAnalyzer6x.java
package com.arno.ik;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Tokenizer;
public class IKAnalyzer6x extends Analyzer{
private boolean useSmart;
public boolean useSmart(){
return useSmart;
}
public void setUseSmart(boolean useSmart){
this.useSmart=useSmart;
}
public IKAnalyzer6x(){
this(false);//IK分词器lucene analyzer接口实现类,默认细粒度切分算法
}
//重写最新版本createComponents;重载analyzer接口,构造分词组件
@Override
protected TokenStreamComponents createComponents(String filedName) {
Tokenizer _IKTokenizer=new IKTokenizer6x(this.useSmart);
return new TokenStreamComponents(_IKTokenizer);
}
public IKAnalyzer6x(boolean useSmart){
super();
this.useSmart=useSmart;
}
}
IKTokenizer6x.java
package com.arno.ik;
import java.io.IOException;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;
public class IKTokenizer6x extends Tokenizer{
//ik分词器实现
private IKSegmenter _IKImplement;
//词元文本属性
private final CharTermAttribute termAtt;
//词元位移属性
private final OffsetAttribute offsetAtt;
//词元分类属性
private final TypeAttribute typeAtt;
//记录最后一个词元的结束位置
private int endPosition;
//构造函数,实现最新的Tokenizer
public IKTokenizer6x(boolean useSmart){
super();
offsetAtt=addAttribute(OffsetAttribute.class);
termAtt=addAttribute(CharTermAttribute.class);
typeAtt=addAttribute(TypeAttribute.class);
_IKImplement=new IKSegmenter(input, useSmart);
}
@Override
public final boolean incrementToken() throws IOException {
//清除所有的词元属性
clearAttributes();
Lexeme nextLexeme=_IKImplement.next();
if(nextLexeme!=null){
//将lexeme转成attributes
termAtt.append(nextLexeme.getLexemeText());
termAtt.setLength(nextLexeme.getLength());
offsetAtt.setOffset(nextLexeme.getBeginPosition(),
nextLexeme.getEndPosition());
//记录分词的最后位置
endPosition=nextLexeme.getEndPosition();
typeAtt.setType(nextLexeme.getLexemeText());
return true;//告知还有下个词元
}
return false;//告知词元输出完毕
}
@Override
public void reset() throws IOException {
super.reset();
_IKImplement.reset(input);
}
@Override
public final void end(){
int finalOffset = correctOffset(this.endPosition);
offsetAtt.setOffset(finalOffset, finalOffset);
}
}
创建好后。如果报错,请查看4个依赖是否已经加载完成和下载的IKAnalyzer2012_u6.jar是否引入
4、创建5个不同的分词器进行分词测试
package com.arno.test;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.cn.smart.SmartChineseAnalyzer;
import org.apache.lucene.analysis.core.SimpleAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import com.arno.ik.IKAnalyzer6x;
/*
* 通过不同分词器analyzer实现对同一个
* 字符串的分词计算查看结果
*/
public class AnalyzerTest {
//编写静态方法,实现打印分词词项的逻辑
public static void printResult(Analyzer analyzer,String str) throws Exception{
//将str转化成字符串输入流
StringReader reader=new StringReader(str);
//anlyzer就是lucene的分词器接口,从analyzer中获取分词的流
//TokenStream
TokenStream tokenStream = analyzer.
tokenStream("test", reader);
tokenStream.reset();
//打印词项,拿到字符属性对象
CharTermAttribute attr=
tokenStream.getAttribute(CharTermAttribute.class);
//挨个打印词项
while(tokenStream.incrementToken()){
System.out.println(attr.toString());
}
}
public static void main(String[] args) throws Exception{
//准备lucene提供的各种分词器对象
Analyzer a1=new SmartChineseAnalyzer();
Analyzer a2=new WhitespaceAnalyzer();
Analyzer a3=new SimpleAnalyzer();
Analyzer a4=new StandardAnalyzer();
Analyzer a5=new IKAnalyzer6x();
String msg="大部分年轻人,现在都喜欢在,业余时间玩王者荣耀";
System.out.println("**********智能中文分词器**********");
printResult(a1, msg);
System.out.println("**********IK分词器**********");
printResult(a5, msg);
System.out.println("**********空格分词器**********");
printResult(a2, msg);
System.out.println("**********简单分词器**********");
printResult(a3, msg);
System.out.println("**********标准分词器**********");
printResult(a4, msg);
}
}