一、思路分析
如何实现近实时搜索呢?每次更新完索引都commit?那恐怕太浪费资源了,当数据量非常庞大的时候,几乎不可能。这里有两种方案。
- 使用SearchManager来管理IndexSearcher对象,当发现索引更新之后,searchManager会调用maybeReopen()方法,来获取最新的IndexSearcher对象,索引search()就是最新的结果了。
- 使用NRTManager和SearcherManager,NRTManager会定时执行maybeReopen()方法来保持最新的索引。它有一个NRTManagerReopenThread类,这个类是一个线程类,用来跟踪索引内存的变化。
二、使用SearcherManager
package com.wsy;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.Executors;
public class IndexUtilPlus {
private String[] ids = {"1", "2", "3", "4", "5", "6"};
private String[] emails = {"a@a.com", "b@b.com", "c@c.com", "d@d.com", "e@e.com", "f@f.com"};
private String[] contents = {"content 1", "content 2", "content 3", "content 4", "content 5", "content 6"};
private int[] attachFiles = {1, 2, 3, 4, 5, 6};
private String[] names = {"qianyi", "zhaoer", "zhangsan", "lisi", "wangwu", "liuliu"};
private Directory directory = null;
private Map<String, Float> scores = new HashMap<>();
private Date[] dates = null;
private SearcherManager searcherManager;
private IndexWriter indexWriter;
public IndexUtilPlus() {
try {
InitDate();
directory = FSDirectory.open(new File("E:\\Lucene\\IndexLibrary"));
indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35)));
searcherManager = new SearcherManager(directory, new SearcherWarmer() {
// 索引更新的时候,需要重新获取IndexSearcher,就会调用这个方法
// 执行maybeReopen()方法的时候就会执行warm()方法,在warm()方法里可以对资源进行控制
@Override
public void warm(IndexSearcher indexSearcher) throws IOException {
System.out.println("index has changed");
}
}, Executors.newCachedThreadPool());
} catch (IOException e) {
e.printStackTrace();
}
}
private void InitDate() {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
dates = new Date[ids.length];
for (int i = 0; i < ids.length; i++) {
try {
dates[i] = simpleDateFormat.parse("2018-10-" + i + 1);
} catch (ParseException e) {
e.printStackTrace();
}
}
}
public void index() {
IndexWriter indexWriter = null;
try {
indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35)));
indexWriter.deleteAll();
Document document = null;
for (int i = 0; i < ids.length; i++) {
document = new Document();
document.add(new Field("id", ids[i], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
document.add(new Field("email", emails[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
document.add(new Field("content", contents[i], Field.Store.NO, Field.Index.ANALYZED));
document.add(new Field("name", names[i], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
String emailType = emails[i].substring(emails[i].lastIndexOf("@") + 1);
if (scores.containsKey(emailType)) {
document.setBoost(scores.get(emailType));
} else {
document.setBoost(0.1f);
}
indexWriter.addDocument(document);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (indexWriter != null) {
try {
indexWriter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
public void search() {
// 使用searcherManager来管理indexSearcher,使用acquire()来获取IndexSearcher对象
IndexSearcher indexSearcher = searcherManager.acquire();
try {
// 当searcherManager检测到硬盘上的索引发生变化的时候
// 通过执行maybeReopen()方法来重新加载IndexSearcher
// 执行maybeReopen()方法的时候,会调用SearcherWarmer中的warm()方法
searcherManager.maybeReopen();
// 搜索content域包含“content”的
TermQuery termQuery = new TermQuery(new Term("content", "content"));
TopDocs topDocs = indexSearcher.search(termQuery, 10);
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
Document document = indexSearcher.doc(scoreDoc.doc);
System.out.println(scoreDoc.doc + " " + document.get("name"));
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
// 使用release()来释放IndexSearcher对象
searcherManager.release(indexSearcher);
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
三、使用SearchManager和NRTManager
package com.wsy;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
public class IndexUtilPlus {
private String[] ids = {"1", "2", "3", "4", "5", "6"};
private String[] emails = {"a@a.com", "b@b.com", "c@c.com", "d@d.com", "e@e.com", "f@f.com"};
private String[] contents = {"content 1", "content 2", "content 3", "content 4", "content 5", "content 6"};
private int[] attachFiles = {1, 2, 3, 4, 5, 6};
private String[] names = {"qianyi", "zhaoer", "zhangsan", "lisi", "wangwu", "liuliu"};
private Directory directory = null;
private Map<String, Float> scores = new HashMap<>();
private Date[] dates = null;
private SearcherManager searcherManager;
private NRTManager nrtManager;
private IndexWriter indexWriter;
public IndexUtilPlus() {
try {
InitDate();
directory = FSDirectory.open(new File("E:\\Lucene\\IndexLibrary"));
indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35)));
nrtManager = new NRTManager(indexWriter, new SearcherWarmer() {
@Override
public void warm(IndexSearcher indexSearcher) throws IOException {
System.out.println("index has changed");
}
});
NRTManagerReopenThread nrtManagerReopenThread = new NRTManagerReopenThread(nrtManager, 5.0, 0.025);
nrtManagerReopenThread.setDaemon(true);
nrtManagerReopenThread.setName("NRTManager Reopen Thread");
nrtManagerReopenThread.start();
// true表示允许所有更新
searcherManager = nrtManager.getSearcherManager(true);
} catch (IOException e) {
e.printStackTrace();
}
}
private void InitDate() {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
dates = new Date[ids.length];
for (int i = 0; i < ids.length; i++) {
try {
dates[i] = simpleDateFormat.parse("2018-10-" + i + 1);
} catch (ParseException e) {
e.printStackTrace();
}
}
}
public void index() {
IndexWriter indexWriter = null;
try {
indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_35, new StandardAnalyzer(Version.LUCENE_35)));
indexWriter.deleteAll();
Document document = null;
for (int i = 0; i < ids.length; i++) {
document = new Document();
document.add(new Field("id", ids[i], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
document.add(new Field("email", emails[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
document.add(new Field("content", contents[i], Field.Store.NO, Field.Index.ANALYZED));
document.add(new Field("name", names[i], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
String emailType = emails[i].substring(emails[i].lastIndexOf("@") + 1);
if (scores.containsKey(emailType)) {
document.setBoost(scores.get(emailType));
} else {
document.setBoost(0.1f);
}
indexWriter.addDocument(document);
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (indexWriter != null) {
try {
indexWriter.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
// 使用NRTManager来删除索引
public void deletePlus() {
try {
nrtManager.deleteDocuments(new Term("id","1"));
} catch (IOException e) {
e.printStackTrace();
}
}
// 使用NRTManager来更新索引
public void updatePlus(){
Document document = new Document();
document.add(new Field("id", "11", Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
document.add(new Field("email", emails[0], Field.Store.YES, Field.Index.NOT_ANALYZED));
document.add(new Field("content", contents[0], Field.Store.NO, Field.Index.ANALYZED));
document.add(new Field("name", names[0], Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));
try {
nrtManager.updateDocument(new Term("id", "1"), document);
} catch (IOException e) {
e.printStackTrace();
}
}
public void search() {
// 使用searcherManager来管理indexSearcher,使用acquire()来获取IndexSearcher对象
IndexSearcher indexSearcher = searcherManager.acquire();
try {
// 当searcherManager检测到硬盘上的索引发生变化的时候
// 通过执行maybeReopen()方法来重新加载IndexSearcher
// 执行maybeReopen()方法的时候,会调用SearcherWarmer中的warm()方法
searcherManager.maybeReopen();
// 搜索content域包含“content”的
TermQuery termQuery = new TermQuery(new Term("content", "content"));
TopDocs topDocs = indexSearcher.search(termQuery, 10);
for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
Document document = indexSearcher.doc(scoreDoc.doc);
System.out.println(scoreDoc.doc + " " + document.get("name"));
}
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
// 使用release()来释放IndexSearcher对象
searcherManager.release(indexSearcher);
} catch (IOException e) {
e.printStackTrace();
}
}
}
}