Java搜索工具——Lucene实例总结(一)

最新推荐文章于 2021-02-16 06:04:46 发布

zzc1684

最新推荐文章于 2021-02-16 06:04:46 发布

阅读量120

点赞数

分类专栏： lucene 文章标签： Java搜索工具——Lucene实例总结(一)

本文链接：https://blog.csdn.net/zzc1684/article/details/84699038

版权

lucene 专栏收录该内容

2 篇文章 0 订阅

订阅专栏

参考网页：http://footman265.iteye.com/blog/849744

搞了一天半，终于利用lucene工具Demo完了我想要的功能，这其中包括为数据库建立增量索引、从索引文件根据id删除索引、单字段查询功能、多字段查询功能、多条件查询功能以及查询结果关键字高亮显示的功能。今天晚些的时候把这些功能进行了整理。看样子一时半会还下不了班，就把Demo的结果一一列举下来吧。。。

理论参考：http://lianj-lee.iteye.com/category/69005?show_full=true

Lucene3.0对数据库建立索引：http://269181927.iteye.com/blog/789779

1. 所需要的文件(见附件)

依赖包：

lucene-core-2.4.0.jar lucene工具包

lucene-highlighter-2.4.0.jar 高亮显示工具包

IKAnalyzer2.0.2OBF.jar 分词工具(支持字典分词)

mysql-connector-java-5.0.3-bin 链接mysql驱动

数据表：

pd_ugc.sql(所在数据库为lucenetest)

类文件：

在附件index.rar和test.rar，解压后放入java工程中的src下即可

2. 为数据库建立增量索引

参考网页：http://www.blogjava.net/laoding/articles/279230.html

Java代码

package index;
//--------------------- Change Logs----------------------
// <p>@author zhiqiang.zhang Initial Created at 2010-12-23<p>
//-------------------------------------------------------
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Date;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
//增量索引
/*
* 实现思路:首次查询数据库表所有记录，对每条记录建立索引，并将最后一条记录的id存储到storeId.txt文件中
* 当新插入一条记录时，再建立索引时不必再对所有数据重新建一遍索引，
* 可根据存放在storeId.txt文件中的id查出新插入的数据，只对新增的数据新建索引，并把新增的索引追加到原来的索引文件中
* */
public class IncrementIndex {
public static void main(String[] args) {
try {
IncrementIndex index = new IncrementIndex();
String path = "E:\\workspace2\\Test\\lucene_test\\poiIdext";//索引文件的存放路径
String storeIdPath = "E:\\workspace2\\Test\\lucene_test\\storeId.txt";//存储ID的路径
String storeId = "";
Date date1 = new Date();
storeId = index.getStoreId(storeIdPath);
ResultSet rs = index.getResult(storeId);
System.out.println("开始建立索引。。。。");
index.indexBuilding(path, storeIdPath, rs);
Date date2 = new Date();
System.out.println("耗时："+(date2.getTime()-date1.getTime())+"ms");
storeId = index.getStoreId(storeIdPath);
System.out.println(storeId);//打印出这次存储起来的ID
} catch (Exception e) {
e.printStackTrace();
}
}
public static void buildIndex(String indexFile, String storeIdFile) {
try {
String path = indexFile;//索引文件的存放路径
String storeIdPath = storeIdFile;//存储ID的路径
String storeId = "";
storeId = getStoreId(storeIdPath);
ResultSet rs = getResult(storeId);
indexBuilding(path, storeIdPath, rs);
storeId = getStoreId(storeIdPath);
System.out.println(storeId);//打印出这次存储起来的ID
} catch (Exception e) {
e.printStackTrace();
}
}
public static ResultSet getResult(String storeId) throws Exception {
Class.forName("com.mysql.jdbc.Driver").newInstance();
String url = "jdbc:mysql://localhost:3306/lucenetest";
String userName = "root";
String password = "****";
Connection conn = DriverManager.getConnection(url, userName, password);
Statement stmt = conn.createStatement();
String sql = "select * from pd_ugc";
ResultSet rs = stmt.executeQuery(sql + " where id > '" + storeId + "'order by id");
return rs;
}
public static boolean indexBuilding(String path, String storeIdPath, ResultSet rs) {
try {
Analyzer luceneAnalyzer = new StandardAnalyzer();
// 取得存储起来的ID，以判定是增量索引还是重新索引
boolean isEmpty = true;
try {
File file = new File(storeIdPath);
if (!file.exists()) {
file.createNewFile();
}
FileReader fr = new FileReader(storeIdPath);
BufferedReader br = new BufferedReader(fr);
if (br.readLine() != null) {
isEmpty = false;
}
br.close();
fr.close();
} catch (IOException e) {
e.printStackTrace();
}
//isEmpty=false表示增量索引
IndexWriter writer = new IndexWriter(path, luceneAnalyzer, isEmpty);
String storeId = "";
boolean indexFlag = false;
String id;
String name;
String address;
String citycode;
while (rs.next()) {
id = rs.getInt("id") + "";
name = rs.getString("name");
address = rs.getString("address");
citycode = rs.getString("citycode");
writer.addDocument(Document(id, name, address, citycode));
storeId = id;//将拿到的id给storeId，这种拿法不合理，这里为了方便
indexFlag = true;
}
writer.optimize();
writer.close();
if (indexFlag) {
// 将最后一个的ID存到磁盘文件中
writeStoreId(storeIdPath, storeId);
}
return true;
} catch (Exception e) {
e.printStackTrace();
System.out.println("出错了" + e.getClass() + "\n 错误信息为: " + e.getMessage());
return false;
}
}
public static Document Document(String id, String name, String address, String citycode) {
Document doc = new Document();
doc.add(new Field("id", id, Field.Store.YES, Field.Index.TOKENIZED));
doc.add(new Field("name", name, Field.Store.YES, Field.Index.TOKENIZED));//查询字段
doc.add(new Field("address", address, Field.Store.YES, Field.Index.TOKENIZED));
doc.add(new Field("citycode", citycode, Field.Store.YES, Field.Index.TOKENIZED));//查询字段
return doc;
}
// 取得存储在磁盘中的ID
public static String getStoreId(String path) {
String storeId = "";
try {
File file = new File(path);
if (!file.exists()) {
file.createNewFile();
}
FileReader fr = new FileReader(path);
BufferedReader br = new BufferedReader(fr);
storeId = br.readLine();
if (storeId == null || storeId == "") storeId = "0";
br.close();
fr.close();
} catch (Exception e) {
e.printStackTrace();
}
return storeId;
}
// 将ID写入到磁盘文件中
public static boolean writeStoreId(String path, String storeId) {
boolean b = false;
try {
File file = new File(path);
if (!file.exists()) {
file.createNewFile();
}
FileWriter fw = new FileWriter(path);
PrintWriter out = new PrintWriter(fw);
out.write(storeId);
out.close();
fw.close();
b = true;
} catch (IOException e) {
e.printStackTrace();
}
return b;
}
}

3. 索引操作

Java代码

package index;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocCollector;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleFragmenter;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.mira.lucene.analysis.IK_CAnalyzer;
public class IndexUtils {
//0. 创建增量索引
public static void buildIndex(String indexFile, String storeIdFile) {
IncrementIndex.buildIndex(indexFile, storeIdFile);
}
//1. 单字段查询
@SuppressWarnings("deprecation")
public static List<IndexResult> queryByOneKey(IndexSearcher indexSearcher, String field,
String key) {
try {
Date date1 = new Date();
QueryParser queryParser = new QueryParser(field, new StandardAnalyzer());
Query query = queryParser.parse(key);
Hits hits = indexSearcher.search(query);
Date date2 = new Date();
System.out.println("耗时：" + (date2.getTime() - date1.getTime()) + "ms");
List<IndexResult> list = new ArrayList<IndexResult>();
for (int i = 0; i < hits.length(); i++) {
list.add(getIndexResult(hits.doc(i)));
}
return list;
} catch (ParseException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
//2. 多条件查询。这里实现的是and操作
//注：要查询的字段必须是index的
//即doc.add(new Field("pid", rs.getString("pid"), Field.Store.YES,Field.Index.TOKENIZED));
@SuppressWarnings("deprecation")
public static List<IndexResult> queryByMultiKeys(IndexSearcher indexSearcher, String[] fields,
String[] keys) {
try {
BooleanQuery m_BooleanQuery = new BooleanQuery();
if (keys != null && keys.length > 0) {
for (int i = 0; i < keys.length; i++) {
QueryParser queryParser = new QueryParser(fields[i], new StandardAnalyzer());
Query query = queryParser.parse(keys[i]);
m_BooleanQuery.add(query, BooleanClause.Occur.MUST);//and操作
}
Hits hits = indexSearcher.search(m_BooleanQuery);
List<IndexResult> list = new ArrayList<IndexResult>();
for (int i = 0; i < hits.length(); i++) {
list.add(getIndexResult(hits.doc(i)));
}
return list;
}
} catch (ParseException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
//3.高亮显示实现了单条件查询
//可改造为多条件查询
public static List<IndexResult> highlight(IndexSearcher indexSearcher, String key) {
try {
QueryParser queryParser = new QueryParser("name", new StandardAnalyzer());
Query query = queryParser.parse(key);
TopDocCollector collector = new TopDocCollector(800);
indexSearcher.search(query, collector);
ScoreDoc[] hits = collector.topDocs().scoreDocs;
Highlighter highlighter = null;
SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("<font color='red'>",
"</font>");
highlighter = new Highlighter(simpleHTMLFormatter, new QueryScorer(query));
highlighter.setTextFragmenter(new SimpleFragmenter(200));
List<IndexResult> list = new ArrayList<IndexResult>();
Document doc;
for (int i = 0; i < hits.length; i++) {
//System.out.println(hits[i].score);
doc = indexSearcher.doc(hits[i].doc);
TokenStream tokenStream = new StandardAnalyzer().tokenStream("name",
new StringReader(doc.get("name")));
IndexResult ir = getIndexResult(doc);
ir.setName(highlighter.getBestFragment(tokenStream, doc.get("name")));
list.add(ir);
}
return list;
} catch (ParseException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
//4. 多字段查询
@SuppressWarnings("deprecation")
public static List<IndexResult> queryByMultiFileds(IndexSearcher indexSearcher,
String[] fields, String key) {
try {
MultiFieldQueryParser mfq = new MultiFieldQueryParser(fields, new StandardAnalyzer());
Query query = mfq.parse(key);
Hits hits = indexSearcher.search(query);
List<IndexResult> list = new ArrayList<IndexResult>();
for (int i = 0; i < hits.length(); i++) {
list.add(getIndexResult(hits.doc(i)));
}
return list;
} catch (ParseException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
//5. 删除索引
public static void deleteIndex(String indexFile, String id) throws CorruptIndexException,
IOException {
IndexReader indexReader = IndexReader.open(indexFile);
indexReader.deleteDocuments(new Term("id", id));
indexReader.close();
}
//6. 一元分词
@SuppressWarnings("deprecation")
public static String Standard_Analyzer(String str) {
Analyzer analyzer = new StandardAnalyzer();
Reader r = new StringReader(str);
StopFilter sf = (StopFilter) analyzer.tokenStream("", r);
System.out.println("=====StandardAnalyzer====");
System.out.println("分析方法：默认没有词只有字（一元分词）");
Token t;
String results = "";
try {
while ((t = sf.next()) != null) {
System.out.println(t.termText());
results = results + " " + t.termText();
}
} catch (IOException e) {
e.printStackTrace();
}
return results;
}
//7. 字典分词
@SuppressWarnings("deprecation")
public static String ik_CAnalyzer(String str) {
Analyzer analyzer = new IK_CAnalyzer();
Reader r = new StringReader(str);
TokenStream ts = (TokenStream) analyzer.tokenStream("", r);
System.out.println("=====IK_CAnalyzer====");
System.out.println("分析方法:字典分词,正反双向搜索");
Token t;
String results = "";
try {
while ((t = ts.next()) != null) {
System.out.println(t.termText());
results = results + " " + t.termText();
}
} catch (IOException e) {
e.printStackTrace();
}
return results;
}
//在结果中搜索
public static void queryFromResults() {
}
//组装对象
public static IndexResult getIndexResult(Document doc) {
IndexResult ir = new IndexResult();
ir.setId(doc.get("id"));
ir.setName(doc.get("name"));
ir.setAddress(doc.get("address"));
ir.setCitycode(doc.get("citycode"));
return ir;
}
}

查询索引结果对象：IndexResult

Java代码

package index;
public class IndexResult {
private String id;
private String name;
private String address;
private String citycode;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public String getCitycode() {
return citycode;
}
public void setCitycode(String citycode) {
this.citycode = citycode;
}
}

4. 测试类

Java代码

package test;
/**
* $Id$
* Copyright 2009-2010 Oak Pacific Interactive. All rights reserved.
*/
import index.IndexResult;
import index.IndexUtils;
import java.util.Date;
import java.util.List;
import org.apache.lucene.search.IndexSearcher;
public class Test {
//存放索引文件
private static String indexFile = "E:\\workspace2\\Test\\lucene_test\\poiIdext";
//存放id
private static String storeIdFile = "E:\\workspace2\\Test\\lucene_test\\storeId.txt";
public static void main(String[] args) throws Exception {
//0. 创建增量索引
IndexUtils.buildIndex(indexFile, storeIdFile);
IndexSearcher indexSearcher = new IndexSearcher(indexFile);
String key = IndexUtils.ik_CAnalyzer("静安中心");
//1.单字段查询
Date date1 = new Date();
List<IndexResult> list = IndexUtils.queryByOneKey(indexSearcher, "name", key);
Date date2 = new Date();
System.out.println("耗时：" + (date2.getTime() - date1.getTime()) + "ms\n" + list.size()
+ "条=======================================单字段查询");
//printResults(list);
//2.多条件查询
String[] fields = { "name", "citycode" };
String[] keys = { IndexUtils.ik_CAnalyzer("静安中心"), "0000" };
date1 = new Date();
list = IndexUtils.queryByMultiKeys(indexSearcher, fields, keys);
date2 = new Date();
System.out.println("耗时：" + (date2.getTime() - date1.getTime()) + "ms\n" + list.size()
+ "条\n===============================多条件查询");
printResults(list);
//3.高亮显示单字段查询
System.out.println("\n\n");
date1 = new Date();
list = IndexUtils.highlight(indexSearcher, key);
date2 = new Date();
System.out.println("耗时：" + (date2.getTime() - date1.getTime()) + "ms\n" + list.size()
+ "条\n======================================高亮显示");
// printResults(list);
//4. 多字段查询
date1 = new Date();
list = IndexUtils.queryByMultiFileds(indexSearcher, fields, key);
date2 = new Date();
System.out.println("耗时：" + (date2.getTime() - date1.getTime()) + "ms\n" + list.size()
+ "条\n=====================================多字段查询");
// printResults(list);
//5. 删除索引中的字段根据id进行删除
IndexUtils.deleteIndex(indexFile, "123");
}
//打印结果
public static void printResults(List<IndexResult> list) {
if (list != null && list.size() > 0) {
for (int i = 0; i < list.size(); i++) {
System.out.println(list.get(i).getId() + "," + list.get(i).getName() + ","
+ list.get(i).getAddress() + "," + list.get(i).getCitycode()+"--->"+i);
}
}
}
}

5. 其它

全文索引：

目前的情况是，搜索hello,"hello world"、"hi hello, how are you"但"worldhello"显示不出来

默认情况下，QueryParser不支持通配符打头的查询（如，*ook）。不过在Lucene 2.1版本以后，他们可以通过调用QueryParser.setAllowLeadingWildcard( true )的 方法打开这一功能。注意，这是一个开销很大的操作：它需要扫描索引中全部记号的列表，来寻找匹配这个模式的词。(译注：高效支持这种后缀查询的办法是，建立反序的记号表，Lucene没有实现这一模式。)http://www.codechina.org/faq/show/42/

支持空格分词搜索："厕所 26 沈阳" 这是三个词

不支持：“厕所沈阳”这是一个词