lucene-使用自带中文分词器

使用lucene内自带的，在contrib/analyzerslucene-analyzers-2.9.0.jar

即单字分析

package busetoken;

import java.io.IOException;
import java.io.StringReader;

import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.cn.ChineseTokenizer;

public class UseCn {

/**
* @param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
String s="编码规范从根本上解决了程序维护员的难题；规范的编码阅读和理解起来更容易，也可以快速的不费力气的借鉴别人的编码。对将来维护你编码的人来说，你的编码越优化，他们就越喜欢你的编码，理解起来也就越快。";
StringReader sr=new StringReader(s);

ChineseTokenizer cjk=new ChineseTokenizer(sr);
Token t=null;

try {
while ((t=cjk.next())!=null){
System.out.print(t.termText()+"|");
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

}

}

效果如下

编|码|规|范|从|根|本|上|解|决|了|程|序|维|护|员|的|难|题|规|范|的|编|码|阅|读|和|理|解|起|来|更|容|易|也|可|以|快|速|的|不|费|力|气|的|借|鉴|别|人|的|编|码|对|将|来|维|护|你|编|码|的|人|来|说|你|的|编|码|越|优|化|他|们|就|越|喜|欢|你|的|编|码|理|解|起|来|也|就|越|快|

建立索引

package bindex;
import java.io.File;
import tool.FileText;
import tool.FileList;
import java.io.*;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.analysis.cn.ChineseAnalyzer;
import org.apache.lucene.store.LockObtainFailedException;
public class FileIndexer {

/**
* @param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
String indexPath ="indexes";
try {
IndexWriter indexWriter = new IndexWriter(indexPath,new ChineseAnalyzer());
String[] files=FileList.getFiles("htmls");
int num=files.length;
for(int i=0;i<num;i++){
Document doc=new Document();
File f=new File(files[i]);

String name=f.getName();
String content=FileText.getText(f);
String path=f.getPath();
Field field=new Field("name",name,Field.Store.YES,Field.Index.TOKENIZED);
doc.add(field);
field=new Field("content",content,Field.Store.YES,Field.Index.TOKENIZED);
doc.add(field);
field=new Field("path",path,Field.Store.YES,Field.Index.NO);
doc.add(field);
indexWriter.addDocument(doc);
System.out.println("File:"+path+name+" indexed!");
}
System.out.println("OK!");
indexWriter.close();
} catch (CorruptIndexException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (LockObtainFailedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}

}

效果如下

File:htmls\wy\360安全中心 - 360安全卫士官网 - 360安全卫士下载免费杀毒木马查杀杀木马电脑安全恶意软件查杀流氓软件查杀.htm360安全中心 - 360安全卫士官网 - 360安全卫士下载免费杀毒木马查杀杀木马电脑安全恶意软件查杀流氓软件查杀.htm indexed!
File:htmls\腾讯首页11.htm腾讯首页11.htm indexed!
OK!

搜"卫士"

package bindex;
import java.io.IOException;
import java.lang.StringBuffer;

import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.document.*;

public class BindexSearcher {

/**
* @param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
String indexPath="indexes";
String searchField="content";
String searchPhrase="卫士";
StringBuffer sb=new StringBuffer("");

try {
IndexSearcher searcher=new IndexSearcher(indexPath);
Term t=new Term(searchField,searchPhrase);
Query q=new TermQuery(t);
Hits hs=searcher.search(q);
int num=hs.length();
for (int i=0;i<num;i++){
Document doc=hs.doc(i);
Field fname=doc.getField("name");
Field fcontent=doc.getField("content");
sb.append("name:\n");
sb.append(fname.stringValue()+"\n");
sb.append("content:\n");
sb.append(fcontent.stringValue().substring(0, 100)+"\n");
sb.append("------------"+"\n");
}
searcher.close();
System.out.println(sb);
} catch (CorruptIndexException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}

}

效果是无法找到

搜索"卫"，使用单字，找到

package bindex;
import java.io.IOException;
import java.lang.StringBuffer;

import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.document.*;

public class BindexSearcher {

/**
* @param args
*/
public static void main(String[] args) {
// TODO Auto-generated method stub
String indexPath="indexes";
String searchField="content";
String searchPhrase="卫";
StringBuffer sb=new StringBuffer("");

try {
IndexSearcher searcher=new IndexSearcher(indexPath);
Term t=new Term(searchField,searchPhrase);
Query q=new TermQuery(t);
Hits hs=searcher.search(q);
int num=hs.length();
for (int i=0;i<num;i++){
Document doc=hs.doc(i);
Field fname=doc.getField("name");
Field fcontent=doc.getField("content");
sb.append("name:\n");
sb.append(fname.stringValue()+"\n");
sb.append("content:\n");
sb.append(fcontent.stringValue().substring(0, 100)+"\n");
sb.append("------------"+"\n");
}
searcher.close();
System.out.println(sb);
} catch (CorruptIndexException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}

}

效果如下

name:
360安全中心 - 360安全卫士官网 - 360安全卫士下载免费杀毒木马查杀杀木马电脑安全恶意软件查杀流氓软件查杀.htm
content:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3c.org/TR/1999/REC-html4
------------
name:
腾讯首页11.htm
content:
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3c.org/TR/1999/REC-html4
------------