freebsd下安装sphinx及其配置文件和分布式检索

安装sphinx

CSFT介绍
CSFT，全称为CoreSeek Fulltext Search Server，也就是CoreSeek 全文检索服务器。Sphinx默认不支持中文索引及检索，CSFT是在Sphinx基础上开发的全文检索软件，按照GPLv2协议发行。Coreseek (http://www.coreseek.com) 为sphinx在中国地区的用户提供支持服务。
到写本文时，CSFT最新版为2009.6.7的3.1rc1版。所有版本发表和修订记录见:http://www.coreseek.cn/products/ft_changelog/

CSFT下载
下载地址:http://www.coreseek.cn/products/ft_down/
选择Source版. Coreseek Fulltext Server(源代码) Coreseek Mmseg(源代码)

安装配置CSFT
1.安装mmseg

1. tar zxvf mmseg-3.1.tar.gz
2.    cd mmseg-3.1
3.    ./configure –prefix=/usr/local/mmseg
4.    make && make install

2.安装csft

1. tar zxvf csft-3.1.tar.gz
2.    cd csft-3.1
3.    ./configure –prefix=/usr/local/sphinx -with-mysql=/usr/local/mysql -with-mysql-includes=/usr/local/mysql/include/mysql -with-mysql-libs=/usr/local/mysql/lib/mysql -with-mmseg-includes=/usr/local/mmseg/include/mmseg -with-mmseg-libs=/usr/local/mmseg/lib -with-mmseg
4.    make && make install

注意：在make时若提示找不到iconv，则在configure结束后在src下的Makefile中查找到LIBS，然后加上 -liconv
3.生成词典
重回到mmseg的源代码目录

1. cd data
2.    mmseg -u unigram.txt
3.    mv  unigram.txt.uni  uni.lib
4.   cp uni.lib /usr/local/sphinx/data/
5. 新建mmseg.ini,输入如下内容:
6. [mmseg]
   merge_number_and_ascii=1;
   number_and_ascii_joint=-;
   compress_space=0;
   seperate_number_ascii=1;
   
   #merge_number_and_ascii: 字母和数字连续出现是非切分
   #number_and_ascii_joint:连接数字和字母可用的符号，如'-' '.' 等
   #compress_space：暂时无效
   #seperate_number_ascii：是否拆分数字，如 1988 -> 1/x 9/x 8/x 8/x

 4.配置索引文件，如下sphinx配置文件,如存在增量索引，请建立下表search_counter:

SET SQL_MODE="NO_AUTO_VALUE_ON_ZERO"; -- -- 数据库: `xmfish_sp3` -- -- -------------------------------------------------------- -- -- 表的结构 `search_counter` -- CREATE TABLE IF NOT EXISTS `search_counter` ( `counterid` int(11) NOT NULL DEFAULT '0', `max_doc_id` int(11) NOT NULL DEFAULT '0', `min_doc_id` int(10) NOT NULL DEFAULT '0', PRIMARY KEY (`counterid`) ) ENGINE=MyISAM DEFAULT CHARSET=gbk;

 5.建立索引 bin/indexer --all(threadsindex)

 6.测试搜索 bin/search 'keyword';也可以直接跳到7、8进行测试

 7.开启searchd服务 /usr/local/sphinx/bin/searchd &

 8.把sphinxapi.php和sphinx.php放到程序目录下进行测试

 9.随机启动searchd,echo "/usr/local/sphinx/bin/searchd">>/etc/rc.local

 

sphinx配置文件和分布式检索

source tmsgs { type = mysql sql_host = localhost sql_user = root sql_pass = phpwind.net sql_db = pw_test sql_port = 3306 # optional, default is 3306 #sql_sock = /tmp/mysql3307.sock sql_query_pre = SET NAMES gbk sql_query_pre = SET SESSION query_cache_type=OFF sql_query_pre = REPLACE INTO search_counter SELECT 1,MAX(tid)+1,MIN(tid) FROM pw_tmsgs sql_query_range = SELECT min_doc_id, max_doc_id FROM search_counter WHERE counterid = 1 sql_range_step = 1000 sql_query = SELECT th.tid,th.subject,th.authorid,th.postdate,th.lastpost,th.fid,th.digest,th.hits,th.replies,t.content FROM pw_threads th LEFT JOIN pw_tmsgs t USING(tid) WHERE th.tid >= $start AND th.tid < $end sql_attr_uint = authorid sql_attr_uint = hits sql_attr_uint = replies sql_attr_uint = fid sql_attr_timestamp = postdate sql_attr_timestamp = lastpost sql_attr_uint = digest } source addtmsgs { type = mysql sql_host = localhost sql_user = root sql_pass = phpwind.net sql_db = pw_test sql_port = 3306 # optional, default is 3306 #sql_sock = /tmp/mysql3307.sock sql_query_pre = SET NAMES gbk sql_query_pre = SET SESSION query_cache_type=OFF sql_query_range = SELECT max_doc_id, max_doc_id+100000 FROM search_counter WHERE counterid = 1 sql_range_step = 100000 sql_query = SELECT th.tid,th.subject,th.authorid,th.postdate,th.lastpost,th.fid,th.digest,th.hits,th.replies,t.content FROM pw_threads th LEFT JOIN pw_tmsgs t USING(tid) WHERE th.tid >= $start AND th.tid < $end sql_attr_uint = authorid sql_attr_uint = hits sql_attr_uint = replies sql_attr_uint = fid sql_attr_timestamp = postdate sql_attr_timestamp = lastpost sql_attr_uint = digest sql_query_post = REPLACE INTO search_counter SELECT 1,MAX(tid)+1,MIN(tid) FROM pw_tmsgs #sql_attr_uint = tid } source threads { type = mysql sql_host = localhost sql_user = root sql_pass = phpwind.net sql_db = pw_test sql_port = 3306 # optional, default is 3306 #sql_sock = /tmp/mysql3307.sock sql_query_pre = SET NAMES gbk sql_query_pre = SET SESSION query_cache_type=OFF sql_query_pre = REPLACE INTO search_counter SELECT 3,MAX(tid)+1,MIN(tid) FROM pw_threads sql_query_range = SELECT min_doc_id, max_doc_id FROM search_counter WHERE counterid = 3 sql_range_step = 1000 sql_query = SELECT th.tid,th.subject,th.authorid,th.postdate,th.lastpost,th.fid,th.digest,th.hits,th.replies FROM pw_threads th WHERE th.tid >= $start AND th.tid < $end sql_attr_uint = authorid sql_attr_uint = hits sql_attr_uint = replies sql_attr_uint = fid sql_attr_timestamp = postdate sql_attr_timestamp = lastpost sql_attr_uint = digest } source addthreads { type = mysql sql_host = localhost sql_user = root sql_pass = phpwind.net sql_db = pw_test sql_port = 3306 # optional, default is 3306 #sql_sock = /tmp/mysql3307.sock sql_query_pre = SET NAMES gbk sql_query_pre = SET SESSION query_cache_type=OFF sql_query_range = SELECT max_doc_id, max_doc_id+100000 FROM search_counter WHERE counterid = 3 sql_range_step = 100000 sql_query = SELECT th.tid,th.subject,th.authorid,th.postdate,th.lastpost,th.fid,th.digest,th.hits,th.replies FROM pw_threads th WHERE th.tid >= $start AND th.tid < $end sql_attr_uint = authorid sql_attr_uint = hits sql_attr_uint = replies sql_attr_uint = fid sql_attr_timestamp = postdate sql_attr_timestamp = lastpost sql_attr_uint = digest sql_query_post = REPLACE INTO search_counter SELECT 3,MAX(tid)+1,MIN(tid) FROM pw_threads #sql_attr_uint = tid } #会员表主索引 source members { type = mysql sql_host = localhost sql_user = root sql_pass = phpwind.net sql_db = pw_test sql_port = 3306 # optional, default is 3306 #sql_sock = /tmp/mysql3307.sock sql_query_pre = SET NAMES gbk sql_query_pre = SET SESSION query_cache_type=OFF sql_query_pre = REPLACE INTO search_counter SELECT 4,MAX(uid)+1,MIN(uid) FROM pw_members sql_query_range = SELECT min_doc_id, max_doc_id FROM search_counter WHERE counterid = 4 sql_range_step = 1000 sql_query = SELECT m.uid,username,groupid,email,oicq,msn,introduce,signature,regdate,onlineip,thisvisit,postnum,lastpost FROM pw_members m LEFT JOIN pw_memberdata md ON md.uid=m.uid WHERE m.uid >= $start AND m.uid < $end sql_attr_uint = groupid sql_attr_uint = username sql_attr_uint = postnum sql_attr_timestamp = regdate sql_attr_timestamp = thisvisit sql_attr_timestamp = lastpost } #会员表增量索引 source addmembers { type = mysql sql_host = localhost sql_user = root sql_pass = phpwind.net sql_db = pw_test sql_port = 3306 # optional, default is 3306 #sql_sock = /tmp/mysql3307.sock sql_query_pre = SET NAMES gbk sql_query_pre = SET SESSION query_cache_type=OFF sql_query_range = SELECT max_doc_id, max_doc_id+100000 FROM search_counter WHERE counterid = 4 sql_range_step = 100000 sql_query = SELECT m.uid,username,groupid,email,oicq,msn,introduce,signature,regdate,onlineip,thisvisit,postnum,lastpost FROM pw_members m LEFT JOIN pw_memberdata md ON md.uid=m.uid WHERE m.uid >= $start AND m.uid < $end sql_attr_uint = groupid sql_attr_uint = username sql_attr_uint = postnum sql_attr_timestamp = regdate sql_attr_timestamp = thisvisit sql_attr_timestamp = lastpost sql_query_post = REPLACE INTO search_counter SELECT 4,MAX(uid)+1,MIN(uid) FROM pw_members } #会员帖子签名主索引 source signatures { type = mysql sql_host = localhost sql_user = root sql_pass = phpwind.net sql_db = pw_test sql_port = 3306 # optional, default is 3306 #sql_sock = /tmp/mysql3307.sock sql_query_pre = SET NAMES gbk sql_query_pre = SET SESSION query_cache_type=OFF sql_query_pre = REPLACE INTO search_counter SELECT 5,MAX(uid)+1,MIN(uid) FROM pw_members sql_query_range = SELECT min_doc_id, max_doc_id FROM search_counter WHERE counterid = 5 sql_range_step = 1000 sql_query = SELECT m.uid,username,groupid,signature,regdate,onlineip,thisvisit,postnum,lastpost FROM pw_members m LEFT JOIN pw_memberdata md ON md.uid=m.uid WHERE m.uid >= $start AND m.uid < $end sql_attr_uint = groupid sql_attr_uint = username sql_attr_uint = postnum sql_attr_timestamp = regdate sql_attr_timestamp = thisvisit sql_attr_timestamp = lastpost } #会员帖子签名增量索引 source addsignatures { type = mysql sql_host = localhost sql_user = root sql_pass = phpwind.net sql_db = pw_test sql_port = 3306 # optional, default is 3306 #sql_sock = /tmp/mysql3307.sock sql_query_pre = SET NAMES gbk sql_query_pre = SET SESSION query_cache_type=OFF sql_query_range = SELECT max_doc_id, max_doc_id+100000 FROM search_counter WHERE counterid = 45 sql_range_step = 100000 sql_query = SELECT m.uid,username,groupid,signature,regdate,onlineip,thisvisit,postnum,lastpost FROM pw_members m LEFT JOIN pw_memberdata md ON md.uid=m.uid WHERE m.uid >= $start AND m.uid < $end sql_attr_uint = groupid sql_attr_uint = username sql_attr_uint = postnum sql_attr_timestamp = regdate sql_attr_timestamp = thisvisit sql_attr_timestamp = lastpost sql_query_post = REPLACE INTO search_counter SELECT 5,MAX(uid)+1,MIN(uid) FROM pw_members } index tmsgsindex { source = tmsgs path = D:/pwsc/data/tmsgs docinfo = extern charset_type = zh_cn.gbk #min_prefix_len = 0 #min_infix_len = 2 #ngram_len = 2 charset_dictpath = D:/pwsc/dict min_prefix_len = 0 min_infix_len = 0 min_word_len = 2 } index addtmsgsindex { source = addtmsgs path = D:/pwsc/data/addtmsgs docinfo = extern charset_type = zh_cn.gbk #min_infix_len = 2 #ngram_len = 2 charset_dictpath = D:/pwsc/dict min_prefix_len = 0 min_infix_len = 0 min_word_len = 2 } index threadsindex { source = threads path = D:/pwsc/data/threads docinfo = extern charset_type = zh_cn.gbk #min_prefix_len = 0 #min_infix_len = 2 #ngram_len = 2 charset_dictpath = D:/pwsc/dict min_prefix_len = 0 min_infix_len = 0 min_word_len = 2 } index addthreadsindex { source = addthreads path = D:/pwsc/data/addthreads docinfo = extern charset_type = zh_cn.gbk #min_infix_len = 2 #ngram_len = 2 charset_dictpath = D:/pwsc/dict min_prefix_len = 0 min_infix_len = 0 min_word_len = 2 } index membersindex { source = members path = D:/pwsc/data/members docinfo = extern charset_type = zh_cn.gbk #min_prefix_len = 0 #min_infix_len = 2 #ngram_len = 2 charset_dictpath = D:/pwsc/dict min_prefix_len = 0 min_infix_len = 0 min_word_len = 2 } index addmembersindex { source = addmembers path = D:/pwsc/data/addmembers docinfo = extern charset_type = zh_cn.gbk #min_prefix_len = 0 #min_infix_len = 2 #ngram_len = 2 charset_dictpath = D:/pwsc/dict min_prefix_len = 0 min_infix_len = 0 min_word_len = 2 } index signaturesindex { source = signatures path = D:/pwsc/data/signatures docinfo = extern charset_type = zh_cn.gbk #min_prefix_len = 0 #min_infix_len = 2 #ngram_len = 2 charset_dictpath = D:/pwsc/dict min_prefix_len = 0 min_infix_len = 0 min_word_len = 2 } index addsignaturesindex { source = addsignatures path = D:/pwsc/data/addsignatures docinfo = extern charset_type = zh_cn.gbk #min_prefix_len = 0 #min_infix_len = 2 #ngram_len = 2 charset_dictpath = D:/pwsc/dict min_prefix_len = 0 min_infix_len = 0 min_word_len = 2 } indexer { mem_limit = 128M } searchd { port = 3312 log = D:/pwsc/logs/searchd.log query_log = D:/pwsc/logs/query.log read_timeout = 5 max_children = 30 pid_file = D:/pwsc/logs/searchd.pid max_matches = 1000 # seamless_rotate = 1 preopen_indexes = 0 unlink_old = 1 }

 

分布式检索

index test{

    type=distributed

    agent=localhost:3312:index1,index2....

    agent=192.168.1.2:3312:index3,index4.....

    agent_connect_timeout=1000

    agent_query_timeout=3000

}