lucene+IKAnalyzer实现中文纯文本检索系统

最新推荐文章于 2022-08-29 19:56:57 发布

weixin_30305735

最新推荐文章于 2022-08-29 19:56:57 发布

阅读量157

点赞数

文章标签： java 开发工具 ui

原文链接：http://www.cnblogs.com/ouym/p/6124087.html

版权

首先IntelliJ IDEA中搭建Maven项目（web）：spring+SpringMVC+Lucene+IKAnalyzer

spring+SpringMVC搭建项目可以参考我的博客

整合Lucene 4.9.0

pom.xml添加lucene依赖

properties标签添加<lucene.version>4.9.0</lucene.version>

dependencies添加：

 1 <!-- lucene start -->
 2       <dependency>
 3           <groupId> org.apache.lucene</groupId>
 4           <artifactId>lucene-analyzers-common</artifactId>
 5           <version> ${lucene.version}</version>
 6       </dependency>
 7 
 8       <dependency>
 9           <groupId> org.apache.lucene</groupId>
10           <artifactId>lucene-core</artifactId>
11           <version> ${lucene.version}</version>
12       </dependency>
13 
14       <dependency>
15           <groupId> org.apache.lucene</groupId>
16           <artifactId>lucene-highlighter</artifactId>
17           <version> ${lucene.version}</version>
18       </dependency>
19 
20       <dependency>
21           <groupId> org.apache.lucene</groupId>
22           <artifactId>lucene-queryparser</artifactId>
23           <version> ${lucene.version}</version>
24       </dependency>
25 <!-- lucene end -->

整合IKAnalyzer 2012FF_hf1，中文分词器的版本要和Lucene的版本对应，Lucene 4.X对应IKAnalyzer 2012FF版本

maven依赖配置参考我的博客

将IKAnalyzer的配置文件考到resources目录里如图：

IKAnalyzer.cfg.xml可以配置词典以及停用词点，其它文件的为自定义停用词典

 1 <?xml version="1.0" encoding="UTF-8"?>
 2 <!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">  
 3 <properties>  
 4     <comment>IK Analyzer 扩展配置</comment>
 5     <!--用户可以在这里配置自己的扩展字典 
 6     <entry key="ext_dict">ext.dic;</entry> 
 7     -->
 8     <!--用户可以在这里配置自己的扩展停止词字典-->
 9     <entry key="ext_stopwords">classpath:stopword.dic;classpath:x-noise-charactor.dic;classpath:x-noise-word.dic;</entry>
10     
11 </properties>

好了，到此项目就配置好了，接下来进行Lucene的核心操作——建索引和检索

IndexService.java

 1 package com.ssm.demo.web.service;
 2 
 3 import com.ssm.demo.core.dto.DocDto;
 4 import org.springframework.ui.Model;
 5 
 6 import java.util.List;
 7 
 8 /**
 9  * Describe: sevice接口
10  * Author: ouym
11  * Created Date: 2016/11/30.
12  */
13 public interface IndexService {
14 
15     /**
16      * 构建索引，传入参数：文档路径
17      * @param path
18      * @return
19      */
20     public boolean createIndex(String path);
21 
22 
23     /**
24      * 通过query查询索引
25      * @param query
26      */
27     public List<DocDto> searchIndex(String query, Model model);
28 }

IndexServiceImpl.java

  1 package com.ssm.demo.web.service.impl;
  2 
  3 import com.ssm.demo.core.constants.MyConstant;
  4 import com.ssm.demo.core.dto.DocDto;
  5 import com.ssm.demo.core.util.MyFileUtil;
  6 import com.ssm.demo.web.service.IndexService;
  7 import org.apache.lucene.analysis.Analyzer;
  8 import org.apache.lucene.analysis.standard.StandardAnalyzer;
  9 import org.apache.lucene.document.Document;
 10 import org.apache.lucene.document.Field;
 11 import org.apache.lucene.document.TextField;
 12 import org.apache.lucene.index.DirectoryReader;
 13 import org.apache.lucene.index.IndexWriter;
 14 import org.apache.lucene.index.IndexWriterConfig;
 15 import org.apache.lucene.queryparser.classic.QueryParser;
 16 import org.apache.lucene.search.IndexSearcher;
 17 import org.apache.lucene.search.Query;
 18 import org.apache.lucene.search.ScoreDoc;
 19 import org.apache.lucene.search.highlight.Highlighter;
 20 import org.apache.lucene.search.highlight.QueryScorer;
 21 import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
 22 import org.apache.lucene.store.Directory;
 23 import org.apache.lucene.store.FSDirectory;
 24 import org.apache.lucene.util.Version;
 25 import org.springframework.stereotype.Service;
 26 import org.springframework.ui.Model;
 27 import org.wltea.analyzer.lucene.IKAnalyzer;
 28 
 29 import java.io.File;
 30 import java.nio.file.Paths;
 31 import java.util.ArrayList;
 32 import java.util.Date;
 33 import java.util.List;
 34 
 35 import static com.ssm.demo.core.util.MyFileUtil.*;
 36 
 37 /**
 38  * Describe: description of this class
 39  * Author: ouym
 40  * Created Date: 2016/11/30.
 41  */
 42 @Service("indexService")
 43 public class IndexServiceImpl implements IndexService {
 44 
 45     public boolean createIndex(String path) {
 46         Date date1 = new Date();
 47         List<File> fileList = getFileList(path);
 48         File indexFile = new File(MyConstant.INDEX_PATH);
 49         //避免重复索引
 50         if (indexFile.exists()){
 51             MyFileUtil.deleteDir(indexFile);
 52         }else {
 53             indexFile.mkdirs();
 54         }
 55         String content="";
 56         Analyzer analyzer = null;
 57         Directory directory = null;
 58         IndexWriter indexWriter = null;
 59 
 60         for (File file : fileList) {
 61             content = "";
 62             //获取文件后缀,只对.doc和.txt文件建索引
 63             String type = file.getName().substring(file.getName().lastIndexOf(".")+1);
 64             if("txt".equalsIgnoreCase(type)){
 65                 content += txt2String(file);
 66             }else if("doc".equalsIgnoreCase(type)){
 67                 content += doc2String(file);
 68             }
 69 
 70             try{
 71 
 72                 //使用第三方中文分词器IKAnalyzer
 73                 analyzer = new IKAnalyzer(true);
 74                 directory = FSDirectory.open(new File(MyConstant.INDEX_PATH));
 75                 IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT,analyzer);
 76                 indexWriter = new IndexWriter(directory, config);
 77 
 78                 Document document = new Document();
 79                 document.add(new TextField("filename", file.getName(), Field.Store.YES));
 80                 document.add(new TextField("content", content, Field.Store.YES));
 81                 document.add(new TextField("path", file.getPath(), Field.Store.YES));
 82                 indexWriter.addDocument(document);
 83                 indexWriter.commit();
 84                 indexWriter.close();
 85 
 86             }catch(Exception e){
 87                 e.printStackTrace();
 88             }
 89             content = "";
 90         }
 91         Date date2 = new Date();
 92         System.out.println("创建索引-----耗时：" + (date2.getTime() - date1.getTime()) + "ms\n");
 93         return false;
 94     }
 95 
 96     public List<DocDto> searchIndex(String queryStr,Model model) {
 97 
 98         Date date1 = new Date();
 99         Analyzer analyzer = null;
100         Directory directory = null;
101         IndexWriter indexWriter = null;
102         String prefixHTML = "<font color='red'>";
103         String suffixHTML = "</font>";
104         List<DocDto> docDtoList = new ArrayList<>();
105         try{
106             directory = FSDirectory.open(new File(MyConstant.INDEX_PATH));
107             //analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
108             analyzer = new IKAnalyzer(true);
109             DirectoryReader ireader = DirectoryReader.open(directory);
110             IndexSearcher isearcher = new IndexSearcher(ireader);
111 
112             QueryParser parser = new QueryParser(Version.LUCENE_CURRENT,"content", analyzer);
113             Query query = parser.parse(queryStr);
114 
115             ScoreDoc[] hits = isearcher.search(query, null, 10).scoreDocs;
116             //ScoreDoc[] hits = isearcher.search(query, 10).scoreDocs;
117 
118             for (int i = 0; i < hits.length; i++) {
119                 DocDto docDto = new DocDto();
120                 Document hitDoc = isearcher.doc(hits[i].doc);
121                 //自动摘要，查询关键词高亮
122                 SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter(prefixHTML, suffixHTML);
123                 Highlighter highlighter = new Highlighter(simpleHTMLFormatter,new QueryScorer(query));
124                 String highLightText = highlighter.getBestFragment(analyzer,"content",hitDoc.get("content"));
125 
126                 docDto.setDocName(hitDoc.get("filename"));
127                 String path = hitDoc.get("path");
128                 path = path.replaceAll("\\\\", "/");
129                 docDto.setDocPath(path);
130                 docDto.setDocAbstract(highLightText+"...");
131                 docDtoList.add(docDto);
132             }
133             ireader.close();
134             directory.close();
135         }catch(Exception e){
136             e.printStackTrace();
137         }
138         Date date2 = new Date();
139         //System.out.println("查看索引-----耗时：" + (date2.getTime() - date1.getTime()) + "ms\n");
140         model.addAttribute("spendTimes",(date2.getTime() - date1.getTime()));
141         return docDtoList;
142     }
143 }

省略了一些常量类和自定义的工具类，接下来只需要在controller里面调用service就行了

 1 @RequestMapping("/index")
 2     public String index(@RequestParam("wd")String wd, Model model){
 3 
 4         //建立索引
 5         indexService.createIndex(MyConstant.DATA_PATH);
 6         if (wd.trim().equals("")){
 7             return "redirect:/index/index";
 8         }
 9 
10         List<DocDto> docDtoList = indexService.searchIndex(wd,model);
11         if (!StringUtils.isEmpty(wd)) {
12             model.addAttribute("query",wd);
13         }
14         model.addAttribute("docDtoList",docDtoList);
15         model.addAttribute("listSize",docDtoList.size());
16         return "result";
17     }