搜索学习——Lucene简介

从今天开始总结一下这段时间以来关于JAVA搜索方面的技术。要讲搜索肯定要从Lucene讲起。Lucene到底是什么？是一个应用程序吗？还是独立的搜索服务器呢？都不是。所谓Lucene，其实可以理解为就是一堆使用JAVA编程，并提供搜索功能的API组件，跟官方一点的说法：Lucene是一个基于Java的全文检索工具包，提供应用程序索引和搜索的功能。
Lucene的应用主要可分为以下三个方向：
一、文本的检索
二、数据库数据的搜索

三、网站信息的搜索（百度干的事情）

Lucene的处理流程是这样的：

首先，对文件或者数据库等需要建立索引的数据进行索引的创建，创建IndexWriter，IndexWriter中定义了索引的一些基本参数，通过IndexWriter创建索引，然后把数据通过Document类添加到索引中，这样对数据创建索引的过程就完成了。

然后用户发起一个Query请求，IndexSearcher类通过search方法搜索Lucene的Index索引，并返回给用户。

代码如下：

public static void main(String args[]){
    System.out.println("hello word");
}

public class Indexer {

    private IndexWriter writer;

    @SuppressWarnings("deprecation")
    public Indexer(String indexDir) throws IOException {
        Directory dir = FSDirectory.open(new File(indexDir));
        Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_33);
        writer = new IndexWriter(dir,analyzer,true,IndexWriter.MaxFieldLength.UNLIMITED);
    }

    public void close() throws IOException {
        writer.close();
    }

    public int index(String dataDir, FileFilter filter) throws Exception {
        File[] files = new File(dataDir).listFiles();
        for(File f : files) {
            if(!f.isDirectory()&&!f.isHidden()&&f.exists()&&f.canRead()&&(filter==null||filter.accept(f))) {
                indexFile(f);
            }
        }
        return writer.numDocs();
    }

    private static class TextFilesFilter implements FileFilter {
        public boolean accept(File path) {
            return path.getName().toLowerCase().endsWith(".doc");
        }
    }

    protected Document getDocument(File f) throws Exception {
        Document doc = new Document();
        doc.add(new Field("文档",new FileReader(f)));
        doc.add(new Field("文件名",f.getName(),Field.Store.YES, Field.Index.NOT_ANALYZED));
        doc.add(new Field("路径",f.getCanonicalPath(),Field.Store.YES, Field.Index.NOT_ANALYZED));
        return doc;
    }

    // 将结果放入Lucene索引中
    private void indexFile(File f) throws Exception {
        System.out.println("Indexing " + f.getCanonicalPath());
        Document doc = getDocument(f);
        writer.addDocument(doc);
    }

    public static void main(String[] args) throws Exception {
        String indexDir = "/home/zhangjian/lucene/test2";
        String dataDir = "/home/zhangjian/lucene/test2";
        long start = System.currentTimeMillis();
        Indexer indexer = new Indexer(indexDir);
        int numIndexed;
        try {
            numIndexed = indexer.index(dataDir,new TextFilesFilter());
        }finally {
            indexer.close();
        }
        long end = System.currentTimeMillis();
        System.out.println("搜索到"+numIndexed+" 文件夹共花费 "+(end-start)+" 毫秒");
    }
}

第一步就是创建IndexWriter，然后通过Document类创建Document对象，再对writer进行addDocument()完成索引的添加。

在最新的lucene5.2.1中有了变化，代码如下：

#coding=utf8
import json
js = json.loads('{"\u6728\u6613\u67d0\u95f2\u4eba":"中国"}')
print json.dumps(js)
print json.dumps(js,ensure_ascii=False)

public class Indexer {

    private IndexWriter indexWriter;
    final Path docDir = Paths.get("D://testFile"); // 存储文档的路径

    public Indexer() throws Exception {
        Directory dir = FSDirectory.open(Paths.get("D://indexFile")); // 使用了nio，存储索引的路径
        Analyzer analyzer = new StandardAnalyzer(); // 无参构造函数
        IndexWriterConfig iwc = new IndexWriterConfig(analyzer); // 新的IndexWriter配置类
        iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE); // 创建模式打开
        //iwc.setRAMBufferSizeMB(256.0); // 设置内存缓存的大小，提高索引效率，不过如果修改过大的值，需要修改JVM的内存值
        IndexWriter writer = new IndexWriter(dir, iwc); // 创建IndexWriter
        indexDocs(writer, docDir); // 对文件进行索引
        writer.close();
    }

    public void indexDocs(IndexWriter writer, Path path) throws Exception {
        File[] files = new File(path.toString()).listFiles();
        for(File f : files) {
            if(!f.isDirectory()&&!f.isHidden()) {
                FileFilter filter = new TextFileFilter();
                if(filter.accept(f)) {
                    indexFile(writer,f);
                }
            }
        }
    }

    private class TextFileFilter implements FileFilter {
        @Override
        public boolean accept(File pathname) {
            return pathname.getName().toLowerCase().endsWith(".txt"); // 仅对txt进行索引
        }
    }

    private void indexFile(IndexWriter writer, File f) throws Exception {
        Document doc = getDocument(f);
        writer.addDocument(doc);
    }

    private Document getDocument(File f) throws Exception {
        Document doc = new Document();
        doc.add(new Field("文档",new FileReader(f)));
        doc.add(new Field("文件名",f.getName(),Field.Store.YES, Field.Index.NOT_ANALYZED));
        doc.add(new Field("路径",f.getCanonicalPath(),Field.Store.YES, Field.Index.NOT_ANALYZED));
        doc.add(new StringField("path",f.getCanonicalPath(),Field.Store.YES));
        return doc;
    }

    public static void main(String[] args) {
        try {
            new Indexer();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

下面是查找的代码：

   
   valid();

class wechatCallbackapiTest
{
    public function valid()
    {
        $echoStr = $_GET["echostr"];

        //valid signature , option
        if($this->checkSignature()){
        	echo $echoStr;
        	exit;
        }
    }

    public function responseMsg()
    {
		//get post data, May be due to the different environments
		$postStr = $GLOBALS["HTTP_RAW_POST_DATA"];

      	//extract post data
		if (!empty($postStr)){
                
              	$postObj = simplexml_load_string($postStr, 'SimpleXMLElement', LIBXML_NOCDATA);
                $fromUsername = $postObj->FromUserName;
                $toUsername = $postObj->ToUserName;
                $keyword = trim($postObj->Content);
                $time = time();
                $textTpl = "
   
   
							
    
    
    
    
							
    
    
    
    
							
    
    
     
     %s
    
    
							
    
    
    
    
							
    
    
    
    
							
    
    
     
     0
    
    
							
   
   ";             
				if(!empty( $keyword ))
                {
              		$msgType = "text";
                	$contentStr = "Welcome to wechat world!";
                	$resultStr = sprintf($textTpl, $fromUsername, $toUsername, $time, $msgType, $contentStr);
                	echo $resultStr;
                }else{
                	echo "Input something...";
                }

        }else {
        	echo "";
        	exit;
        }
    }
		
	private function checkSignature()
	{
        $signature = $_GET["signature"];
        $timestamp = $_GET["timestamp"];
        $nonce = $_GET["nonce"];	
        		
		$token = TOKEN;
		$tmpArr = array($token, $timestamp, $nonce);
		sort($tmpArr);
		$tmpStr = implode( $tmpArr );
		$tmpStr = sha1( $tmpStr );
		
		if( $tmpStr == $signature ){
			return true;
		}else{
			return false;
		}
	}
}
?>

public class Searcher {

    String field = "文档";

    public void search(String indexDir, String queryStr) throws IOException, ParseException {
        Directory dir = FSDirectory.open(Paths.get("D://indexFile"));
        IndexReader reader = DirectoryReader.open(dir);
        IndexSearcher is = new IndexSearcher(reader);
        Analyzer analyzer = new StandardAnalyzer();
        QueryParser parser = new QueryParser(field, analyzer);
        Query query = parser.parse(queryStr);
        TopDocs hits = is.search(query,10);
        for(ScoreDoc scoreDoc : hits.scoreDocs) {
            Document doc = is.doc(scoreDoc.doc);
            System.out.println(doc.get("path")); // 找到文件后，输出路径
        }
    }

    public static void main(String[] args) throws Exception {
        new Searcher().search("D://testFile","good");
    }
}