solr使用中文，庖丁分词

1、分析器改造 net.paoding.analysis.analyzer.PaodingTokenizer.java



extends Tokenizer 原来是 extends TokenStream



2、net.paoding.analysis.analyzer.solr.ChineseTokenizerFactory.java



package net.paoding.analysis.analyzer.solr;

import java.io.Reader;

import java.util.Map;



import net.paoding.analysis.analyzer.PaodingTokenizer;

import net.paoding.analysis.analyzer.TokenCollector;

import net.paoding.analysis.analyzer.impl.MaxWordLengthTokenCollector;

import net.paoding.analysis.analyzer.impl.MostWordsTokenCollector;

import net.paoding.analysis.knife.PaodingMaker;



import org.apache.lucene.analysis.TokenStream;

import org.apache.lucene.analysis.Tokenizer;

import org.apache.solr.analysis.BaseTokenizerFactory;



/**

* Created by IntelliJ IDEA. User: ronghao Date: 2007-11-3 Time: 14:40:59 中文切词

* 对庖丁切词的封装

*/

public class ChineseTokenizerFactory extends BaseTokenizerFactory {

/**

* 最多切分 默认模式

*/

public static final String MOST_WORDS_MODE = "most-words";

/**

* 按最大切分

*/

public static final String MAX_WORD_LENGTH_MODE = "max-word-length";

private String mode = null;



public void setMode(String mode) {

if (mode == null || MOST_WORDS_MODE.equalsIgnoreCase(mode)

|| "default".equalsIgnoreCase(mode)) {

this.mode = MOST_WORDS_MODE;

} else if (MAX_WORD_LENGTH_MODE.equalsIgnoreCase(mode)) {

this.mode = MAX_WORD_LENGTH_MODE;

} else {

throw new IllegalArgumentException(

"不合法的分析器Mode 参数设置:"

+ mode);

}

}



@Override

public void init(Map<String,String> args) {

super.init(args);

setMode(args.get("mode"));

}



public Tokenizer create(Reader input) {

return new PaodingTokenizer(input, PaodingMaker.make(),

createTokenCollector());

}



private TokenCollector createTokenCollector() {

if (MOST_WORDS_MODE.equals(mode))

return new MostWordsTokenCollector();

if (MAX_WORD_LENGTH_MODE.equals(mode))

return new MaxWordLengthTokenCollector();

throw new Error("never happened");

}

}



3、下载庖丁 将paoding-analysis.jar改造成paoding-analysis-solr.jar（将1,2覆盖进去这个包）



4、定义庖丁词典 paoding-dic-home.properties



paoding.dic.home=D:/WORK/DOCSET/solr/paoding-analysis-2.0.4-beta/dic/



5、替换原有的分词类 D:\WORK\DOCSET\solr\j-solr1\solr\conf\schema.xml



<fieldtype name="text" class="solr.TextField" positionIncrementGap="100">

<analyzer type="index">

<tokenizer class="net.paoding.analysis.analyzer.solr.ChineseTokenizerFactory" mode="most-words"/>

<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>

<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0"/>

<filter class="solr.LowerCaseFilterFactory"/>

<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>

</analyzer>

<analyzer type="query">

<tokenizer class="net.paoding.analysis.analyzer.solr.ChineseTokenizerFactory" mode="most-words"/>

<filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>

<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>

<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0"/>

<filter class="solr.LowerCaseFilterFactory"/>

<filter class="solr.RemoveDuplicatesTokenFilterFactory"/>

</analyzer>

</fieldtype>