lucene使用

/**
*
*/
package com;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.wltea.analyzer.lucene.IKAnalyzer;
import org.wltea.analyzer.lucene.IKSimilarity;

import com.bean.FtpBean;

/**
* @author Administrator
*
*/
public class IndexDeal{


/**
* 创建索引
*
* @param indexDirPath
* 索引目录
* @param dataDirPath
* 数据文件目录 日志目录
* @throws Exception
*/
public static void createIndex(final String indexDirPath,
final String dataDirPath) {

try {
System.out.println("begin create index ...........");
// 设置索引地址
File indexDir = new File(indexDirPath);
// 设置数据地址
File dataDir = new File(dataDirPath);
// 实例化IKAnalyzer分词器
Analyzer ikAnalyzer = new IKAnalyzer();
// 取得目录下所有Files
File[] dataFiles = dataDir.listFiles();
// 建立indexWrite indexWrite主要作用是添加索引
IndexWriter indexWriter = new IndexWriter(indexDir, ikAnalyzer,
true, IndexWriter.MaxFieldLength.UNLIMITED);
//新添加
//控制多个segment合并的频率，值较大时建立索引速度较快，默认是10，可以在建立索引时设置为100。
//indexWriter.setMergeFactor(100);
//控制写入一个新的segment前内存中保存的document的数目，设置较大的数目可以加快建索引速度，默认为10。
//indexWriter.setMaxBufferedDocs(100);


// 取得程序开启时间
long startTime = System.currentTimeMillis();
// 循环文件
for (int i = 0; i < dataFiles.length; i++) {
// 取出txt后缀的文档
//if (dataFiles[i].isFile()&& dataFiles[i].getName().endsWith(".log")) {
if (dataFiles[i].isFile()) {
System.out.println("Indexing file " + dataFiles[i].getCanonicalPath());

// 读取数据
BufferedReader txtReader = new BufferedReader(
new InputStreamReader(new FileInputStream(
dataFiles[i])));
// Document添加正文
String text = null;
while ((text = txtReader.readLine()) != null) {
// 新建一个Document
Document document = new Document();
// Document添加path
// Field.Store.YES:存储字段值（未分词前的字段值）
// Field.Store.NO:不存储,存储与索引没有关系
// Field.Store.COMPRESS:压缩存储,用于长文本或二进制，但性能受损

// Field.Index.ANALYZED:分词建索引
// Field.Index.ANALYZED_NO_NORMS:分词建索引，但是Field的值不像通常那样被保存，而是只取一个byte，这样节约存储空间
// Field.Index.NOT_ANALYZED:不分词且索引
// Field.Index.NOT_ANALYZED_NO_NORMS:不分词建索引，Field的值去一个byte保存

// TermVector表示文档的条目（由一个Document和Field定位）和它们在当前文档中所出现的次数
// Field.TermVector.YES:为每个文档（Document）存储该字段的TermVector
// Field.TermVector.NO:不存储TermVector
// Field.TermVector.WITH_POSITIONS:存储位置
// Field.TermVector.WITH_OFFSETS:存储偏移量
// Field.TermVector.WITH_POSITIONS_OFFSETS:存储位置和偏移量
// System.out.println("text=["+text);
//document.add(new Field("path", dataFiles[i].getCanonicalPath(), Field.Store.YES,Field.Index.ANALYZED));
// document.add(new Field("contents", text,Field.Store.YES,Field.Index.ANALYZED_NO_NORMS));
document.add(new Field("contents", text,Field.Store.YES, Field.Index.NOT_ANALYZED));
//System.out.println(document.toString());
// 添加索引
indexWriter.addDocument(document);
}

txtReader.close();

}
}
// 优化
indexWriter.optimize();
// 关闭流
indexWriter.close();
long endTime = System.currentTimeMillis();

// 输出程序所用时间
System.out
.println("It takes "
+ (endTime - startTime)
+ " milliseconds to create index for the files in directory "
+ dataDir.getPath());

} catch (Exception e) {
e.printStackTrace();
}
}


/**
*
* @param indexDirectory
* 索引路径
* @param FtpBean
* @return 搜索到的日志内容
* @throws IOException
* @throws ParseException
*/
public static void multiQuery(final String indexDirectory,
FtpBean FtpBean) {

try{
System.out.println("begin search results ........");
Directory dir = FSDirectory.getDirectory(indexDirectory, false);
IndexSearcher indexSearcher = new IndexSearcher(dir);
// 在索引器中使用IKSimilarity相似度评估器
indexSearcher.setSimilarity(new IKSimilarity());

// 构造容器
BooleanQuery query = new BooleanQuery();

//设置最大词条数量
//BooleanQuery.setMaxClauseCount(1024);

String tradeDate = null;
if (!"".equals(FtpBean.getKey1())) {

WildcardQuery q1 = new WildcardQuery(new Term("contents", "*"
+ FtpBean.getKey1() + "*"));

// Query q1 = new TermQuery(new
// Term("contents",logManageBean.getKey1().toLowerCase()));
query.add(q1, BooleanClause.Occur.MUST);

}

if (!"".equals(FtpBean.getKey2())) {

WildcardQuery q2 = new WildcardQuery(new Term("contents", "*"
+ FtpBean.getKey2() + "*"));
query.add(q2, BooleanClause.Occur.MUST);

}
if (!"".equals(FtpBean.getTradeDate())) {

tradeDate = FtpBean.getTradeDate();
WildcardQuery q3 = new WildcardQuery(new Term("contents", "*"
+FtpBean.getTradeDate()+ "*"));
query.add(q3, BooleanClause.Occur.MUST);

}

if (!"".equals(FtpBean.getTradeTime())) {

WildcardQuery q4 = new WildcardQuery(new Term("contents", "*"
+ FtpBean.getTradeTime() + "*"));
query.add(q4, BooleanClause.Occur.MUST);

}

System.out.println("输入的关键字是【" + FtpBean.getKey1() + "】 【"
+ FtpBean.getKey2() + "】 【"
+ FtpBean.getTradeDate() + "】 【"
+ FtpBean.getTradeTime() + "】");

Hits results = indexSearcher.search(query);

System.out.println("[" + results.length() + "] search results is returned");


int size = results.length();

for (int j = 0; j < size; j++) {

System.out.println("【"+j+"】 content=【"+results.doc(j).get("contents")+"】");
}
}catch(Exception e){
e.printStackTrace();
}
}



public static void main(String[] args) {
String indexdir1 = "d:/index";
String indexdir2 = "d:/index2";
String datadir = "d:/logs";
String flag="1";
String datadir3 = "d:/logs3";
String indexdir3 = "d:/index3";

if("1".equals(flag)){
createIndex(indexdir3,datadir3);
}else{
createIndex(indexdir1, datadir);
createIndex(indexdir2, datadir);
mergeIndex(new File(indexdir2),new File(indexdir1),new IKAnalyzer());
}

try{
FtpBean FtpBean = new FtpBean();
FtpBean.setKey1("SQL");
FtpBean.setKey2("");
FtpBean.setTradeDate("2011-01-12");
FtpBean.setTradeTime("");

multiQuery(indexdir1,FtpBean);

}catch(Exception e){
e.printStackTrace();
}
}


/**
*
* 将小索引文件合并到大的索引文件中去
*
* @param from
* 将要合并到to文件的文件
* @param to
* 将from文件合并到该文件
* @param sa
*/
private static void mergeIndex(File from, File to,IKAnalyzer sa) {
IndexWriter indexWriter = null;
// 取得程序开启时间
long startTime = System.currentTimeMillis();
try {
System.out.println("正在合并索引文件!\t ");
indexWriter = new IndexWriter(to, sa, false);
indexWriter.setMergeFactor(100000);
indexWriter.setMaxFieldLength(Integer.MAX_VALUE);
indexWriter.setMaxBufferedDocs(Integer.MAX_VALUE);
indexWriter.setMaxMergeDocs(Integer.MAX_VALUE);
FSDirectory[] fs = { FSDirectory.getDirectory(from, false) };
indexWriter.addIndexes(fs);
indexWriter.optimize();
indexWriter.close();
System.out.println("已完成合并!\t ");
} catch (Exception e) {
System.out.println("合并索引出错！");
e.printStackTrace();
} finally {
try {
if (indexWriter != null)
indexWriter.close();
} catch (Exception e) {

}

}
// 取得程序开启时间
long endTime = System.currentTimeMillis();

// 输出程序所用时间
System.out
.println("It takes "
+ (endTime - startTime)
+ " milliseconds to merge index for the files in directory ");
}


}