lucene 索引实战

Lucene索引实战

——comaple.zhang

声明：本文为原创，转载请注明出处。

1.概述

Lucene 是用来建立索引和查询索引的利器，其查询速度很快。我本人测试的数据位125万条数据每条200字节，返回结果要1.7s内完成。而从结果集中取出120万数据并转换为pojo对象则需要13.8s。网络i/o达到16MB/s。而建立120万条这样的索引数据仅需要1.26m。那么现在就让我们一起来看一下具体的用法吧。

2．概念解释

2.1倒排索引

Lucene是高性能的基于java实现的全文检索工具，可以把他理解为全文检索的api集合。它使用的是倒排文件索引结构。下面举一个例子来说明倒排索引的概念。假如有两篇文章，

Doc1：tom lives inGuangzhou，I live in Guangzhou too。

Doc2：he oncelived in shanghai。

a)由于lucene是基于关键词索引和查询的，所以首先我们要对文章进行分词处理。为了简便我们这里只讨论英文分词，即以空格分隔的单词系列，再去掉小品词即可。

b)单词分词以后统一大小写，还原英文单词原型。如lives lived还原为live

c)去掉标点符号，此时文章变为：

Doc1：[tom] [live] [guangzhou][live] [guangzhou]

Doc2: [he] [live] [shanghai]

d)接下来我们来进行倒排索引，一般索引以这样的格式存储：文章id ：关键字；而倒排索引反过来了如：关键字：包还该关键字的所有文档id。如下所示：

关键字 文章id

Guangzhou doc1

He doc2

I doc1

Live doc1、doc2

Shanghai doc2

Tom doc1

2.2 Document 和 Field

对于要index的文档, 首先要生成抽象的document对象, 然后把需要index的内容加到各个fields中去.Document就是fields的集合, 而fields用来放文档本身和文档的元数据.

field由名字和值组成.如代码：

Document doc = new Document();

String pin = item.getPin();

doc.add(new Field("pin", pin,Store.YES, Index.NOT_ANALYZED));

doc.add(new Field("sku",item.getSku(), Store.YES,Index.ANALYZED));

2.3写入文档

indexWriter.addDocument(doc);

2.4 创建indexwriter

IndexWriter writer = new IndexWriter(dir,getAnalyzer(), true);

3下面一个完整的demo

直接上代码了，这是一个可以实现创建，更新和查询的工具类，其中查询实现了分页读取的操作。

import java.io.File;

import java.util.ArrayList;

import java.util.Iterator;

import java.util.List;

import java.util.concurrent.locks.Lock;

import java.util.concurrent.locks.ReentrantLock;

import org.apache.lucene.analysis.Analyzer;

import org.apache.lucene.analysis.standard.StandardAnalyzer;

import org.apache.lucene.document.Document;

import org.apache.lucene.document.Field;

import org.apache.lucene.document.Field.Index;

import org.apache.lucene.document.Field.Store;

import org.apache.lucene.index.IndexReader;

import org.apache.lucene.index.IndexWriter;

import org.apache.lucene.index.IndexWriterConfig;

import org.apache.lucene.index.Term;

import org.apache.lucene.search.IndexSearcher;

import org.apache.lucene.search.Query;

import org.apache.lucene.search.ScoreDoc;

import org.apache.lucene.search.TermQuery;

import org.apache.lucene.search.TopDocs;

import org.apache.lucene.search.TopScoreDocCollector;

import org.apache.lucene.store.Directory;

import org.apache.lucene.store.NIOFSDirectory;

import org.apache.lucene.util.Version;

import jd.data.migration.dao.BaseDao;

import jd.data.migration.model.UserOrderModel;

/**

*

* @author qt-zhangshengtao

*

*/

public class LuceneDaoImpl implements BaseDao<UserOrderModel>{

//索引文件存放的路径

private final String DBPATH = "e:\\newindex";

//indexwriter

private static IndexWriter indexWriter =null;

//indexwriter用到的存储对象

private static Directory indexDir =null;

//配置对象

private static IndexWriterConfig config =null;

public static Analyzer luceneAnalyzer =null;

private IndexReader reader = null;

private Query query = null;

private IndexSearcher searcher = null;

// 默认每页返回一万

private final int pageCount = 10000;

public static LuceneDaoImpl instance;

private static Lock lock = new ReentrantLock();

private LuceneDaoImpl() {

try {

luceneAnalyzer = new StandardAnalyzer(Version.LUCENE_35);

config = new IndexWriterConfig(Version.LUCENE_35,luceneAnalyzer);

indexDir = new NIOFSDirectory(new File(DBPATH));

indexWriter = new IndexWriter(indexDir,config);

indexWriter.setRAMBufferSizeMB(512);

//indexWriter.setUseCompoundFile(false);

} catch (Exception e) {

System.out.println(e.getMessage());

}

}

public static LuceneDaoImpl getInstance() {

lock.lock();

if (instance == null)

instance = new LuceneDaoImpl();

lock.unlock();

return instance;

}

/**

* 根据单个pin获取userOrder列表

*

* @param pin

* @return

*/

public List<UserOrderModel> getListByPin(String sku) {

try {

List<UserOrderModel> list = newArrayList<UserOrderModel>();

query = new TermQuery(new Term("sku", sku));

reader = IndexReader.open(indexDir);

searcher = new IndexSearcher(reader);

//TopScoreDocCollector result = TopScoreDocCollector.cr

TopDocs docs = searcher.search(query, pageCount);

int totalCount = docs.totalHits;

if (totalCount > pageCount) {

for (int i = 0; i < totalCount / pageCount; i++) {

ScoreDoc[] scoreDocs = docs.scoreDocs;

ScoreDoc lastdoc = null;

for (ScoreDoc scoreDoc : scoreDocs) {

Document document = searcher.doc(scoreDoc.doc);

list.add(new UserOrderModel(document.get("pin"),

document.get("sku")));

lastdoc = scoreDoc;

}

docs = searcher.searchAfter(lastdoc, query, pageCount);

if (docs.scoreDocs.length == 0)

break;

}

}

return list;

} catch (Exception e) {

System.out.println(e.getMessage());

return null;

}

}

/**

* 在lucene中根据传入的list创建索引

*

* @param list

*/

public void createIndex(List<UserOrderModel> list) {

try {

for (Iterator iterator = list.iterator(); iterator.hasNext();) {

UserOrderModel item = (UserOrderModel) iterator.next();

Document doc = new Document();

String pin = item.getPin();

doc.add(new Field("pin", pin, Store.YES, Index.NOT_ANALYZED));

doc.add(new Field("sku", item.getSku(), Store.YES,

Index.ANALYZED));

indexWriter.addDocument(doc);

}

} catch (Exception e) {

System.out.println(e.getMessage());

}

}

/**

* 更新索引

*/

public void updateIndex(List<UserOrderModel> list) {

try {

for (Iterator iterator = list.iterator();iterator.hasNext();) {

UserOrderModel item = (UserOrderModel) iterator.next();

Document doc = new Document();

doc.add(new Field("pin", item.getPin(), Store.YES,

Index.NOT_ANALYZED));

doc.add(new Field("sku", item.getSku().toString(), Store.YES,

Index.ANALYZED_NO_NORMS));

indexWriter.updateDocument(new Term("pin", item.getPin()), doc);

}

} catch (Exception e) {

e.printStackTrace();

}

}

public void close() {

try {

// TODO Auto-generatedmethod stub

indexWriter.commit();

indexWriter.close();

} catch (Exception e) {

System.out.println(e.getMessage());

}

}

public List<UserOrderModel> getListByPin(String[] pin) {

// TODO Auto-generatedmethod stub

return null;

}

}