solr扩展自己的analyzer的tokenfilter

1、analyzer通常一个Tokenizer ， 以及几个 TokenFilter 组成

tokenfiter的概念是词语级别处理过滤器，它的上一层是Tokenizer，字符串级别处理流。

schema.xml中配置是：

<fieldType name="text" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
      <analyzer type="index">
        <tokenizer class="solr.StandardTokenizerFactory"/>
        <filter class="solr.LowerCaseFilterFactory"/>
        <filter class="com.wo.lucene.analysis.NumberEnglishFilterFactory"/>
      </analyzer>
      <analyzer type="query">
        <tokenizer class="solr.StandardTokenizerFactory"/>
        <filter class="solr.LowerCaseFilterFactory"/>
      </analyzer>
    </fieldType>

很容易看出经过Tokenizer切出的term还需要经过多个filter过滤。

2、接下来如何编写自己的filter

NumberEnglishFilter 主要是为了在索引时切分数字+字母，比如"2014show"，“2014”、"show"作为term，这样检索2014或者show都能检索出来。

整个代码不难理解，唯一要注意的地方是：

curTermBuffer = termAtt.buffer().clone();
curTermLength = termAtt.length();

curtermBuffer有可能会比实际的termAtt要大，如果上一次缓存的termAtt长度为10，那么curTermBuffer长度就为10，所以要自己计算termAtt的长度，curTermLength = termAtt.length();

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;


import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.util.Version;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public class NumberEnglishFilter extends TokenFilter {
private static final Logger logger = LoggerFactory.getLogger(NumberEnglishFilter.class);
private char[] curTermBuffer;
// char[]总长度
private int curTermLength;
// 当前切割处,+1表示下一次切割处
private int curGramSize;

        //number和english位置
private List<Integer> positions;
// 相对位置，即在整个输入词的位置
private int tokStart;
// positions所在的位置
private int position;


private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);

        //表示开始位置，结束位置
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);


// private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);


public NumberEnglishFilter(Version matchVersion, TokenStream input) {
super(input);
}


@Override
public boolean incrementToken() throws IOException {
// 表示还有token
while (true) {
// 表示当前token结束，进入下一个token
if (curTermBuffer == null) {
if (!input.incrementToken()) {
return false;
} else {
curTermBuffer = termAtt.buffer().clone();
curTermLength = termAtt.length();
// 放置position list
positions = getPositions(curTermBuffer, curTermLength);
tokStart = offsetAtt.startOffset();
curGramSize = positions.get(0);
position = 0;
}
}
if (curGramSize < curTermLength && curTermLength > 1 && positions.size() > 2) {
try {
position++;
offsetAtt.setOffset(tokStart + curGramSize, tokStart + positions.get(position));
termAtt.copyBuffer(curTermBuffer, curGramSize, positions.get(position) - curGramSize);
curGramSize = positions.get(position);
return true;
} catch (Exception e) {
logger.error(position + "\t" + new String(curTermBuffer));
clearAttributes();
offsetAtt.setOffset(tokStart + 0, tokStart + curTermLength);
termAtt.copyBuffer(curTermBuffer, 0, curTermLength);
curTermBuffer = null;
return true;
}

} else {
clearAttributes();
offsetAtt.setOffset(tokStart + 0, tokStart + curTermLength);
termAtt.copyBuffer(curTermBuffer, 0, curTermLength);
curTermBuffer = null;
return true;
}


}


}

        //记录下number和english位置
public List<Integer> getPositions(char[] term, int length) {
List<Integer> list = new ArrayList<Integer>();
list.add(0);
for (int i = 0; i < length;) {
if (Character.isDigit(term[i])) {
while (++i < length && Character.isDigit(term[i]))
;
list.add(i);
continue;
} else if (i < length && Character.isLetter(term[i])) {
while (++i < length && Character.isLetter(term[i]))
;
list.add(i);
continue;
} else {
list.add(++i); 
}
}
if(!Character.isDigit(term[length-1]) && !Character.isLetter(term[length-1]))
list.add(length);
return list;
}


}