Lucene7.0与HanLP分词器整合索引数据库建立索引文件

HanLP官网：http://hanlp.linrunsoft.com/

GitHup地址：https://github.com/hankcs/HanLP

HanLP插件地址：https://github.com/hankcs/hanlp-lucene-plugin

需要一下jar包

类

package com.kyd.demo.hanLP;

import java.io.IOException;
import java.nio.file.Paths;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.junit.Test;

import com.hankcs.lucene.HanLPAnalyzer;
import com.hankcs.lucene.HanLPIndexAnalyzer;

/**
 * 索引数据库字段建立索引文件
 * 
 * @author zhengzhen
 *
 */
public class JdbcIndexDemo {
	public static void main(String[] args) {
		try {
			Class.forName("com.mysql.jdbc.Driver");
			String url = "jdbc:mysql://192.168.100.69:3306/xxxx?useUnicode=true&characterEncoding=utf8&autoReconnect=true&failOverReadOnly=false";
			String password ="root";
			String userName ="root";
			String sql ="select * from xxxx";
			try (
					Connection conn = DriverManager.getConnection(url,userName,password);
					PreparedStatement sta =conn.prepareStatement(sql);
					ResultSet rs = sta.executeQuery();
					){
				/**
				 * 1.设置索引文件保存路径
				 */
					Directory directory = FSDirectory.open(Paths.get("xxxx_index"));
					/**
					 * 2.创建分词器
					 */
					Analyzer analyzer = new HanLPIndexAnalyzer();
					/**
					 * 3.分词器配置
					 */
					IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer);
					indexWriterConfig.setOpenMode(OpenMode.CREATE);
					/**
					 * 4.创建索引输出流
					 */
					IndexWriter indexWriter = new IndexWriter(directory,indexWriterConfig);
					/**
					 * 5.循环遍历创建索引文档
					 */
					while (rs.next()) {
						/**
						 * 5.1.创建文档
						 */
						Document document = new Document();
						/**
						 * 5.2.添加字段
						 */
						Long id  =rs.getLong("unitId");
						IndexableField unitIdField = new StringField("unitId", id+"",Store.YES);
						document.add(unitIdField);
						
						String title = rs.getString("title");
						if( title != null) {
							IndexableField sectionNameField = new TextField("sectionName", title, Store.YES);
							document.add(sectionNameField);
						}
						
					
						
						String  unitName= rs.getString("unitName");
						if( unitName != null) {
							IndexableField unitNameField = new TextField("unitName", unitName, Store.YES);
							document.add(unitNameField);
						}
						
						
						String  courseName= rs.getString("courseName");
						if(courseName !=null) {
							IndexableField courseNameField = new TextField("courseName", courseName, Store.YES);
							document.add(courseNameField);
						}
						
						
						String  startPage= rs.getString("startPage");
						if(startPage !=null) {
							IndexableField startPageField = new StringField("startPage", startPage, Store.YES);
							document.add(startPageField);
						}
						
						
						String  endPage= rs.getString("startEndPage");
						if(endPage != null) {
							IndexableField endPageField = new StringField("endPage", endPage,Store.YES);
							document.add(endPageField);
						}
					
						
						indexWriter.addDocument(document);
						
					}
					indexWriter.commit();
			} catch (Exception e) {
				e.printStackTrace();
			}
		} catch (ClassNotFoundException e1) {
			
			e1.printStackTrace();
		}
		
	}
	/**
	 * HanLPAnalyzer
	 * 这个分词器对于长词不会切割 ，例如 “中华人民共和国” 是一个长词会保留下来
	 * @throws IOException
	 */
	@Test
	public void hanLPAnalyzerTest() throws IOException {
		String text = "中华人民共和国很辽阔";
		for (int i = 0; i < text.length(); ++i)
		{
		    System.out.print(text.charAt(i) + "" + i + " ");
		}
		System.out.println();
		Analyzer analyzer = new HanLPAnalyzer();
		TokenStream tokenStream = analyzer.tokenStream("field", text);
		tokenStream.reset();
		while (tokenStream.incrementToken())
		{
		    CharTermAttribute attribute = tokenStream.getAttribute(CharTermAttribute.class);
		    // 偏移量
		    OffsetAttribute offsetAtt = tokenStream.getAttribute(OffsetAttribute.class);
		    // 距离
		    PositionIncrementAttribute positionAttr = tokenStream.getAttribute(PositionIncrementAttribute.class);
		    System.out.println(attribute + " " + offsetAtt.startOffset() + " " + offsetAtt.endOffset() + " " + positionAttr.getPositionIncrement());
		}
		/* 输出：
		 * 中0 华1 人2 民3 共4 和5 国6 很7 辽8 阔9 
		 * 中华人民共和国 0 7 1
		 * 很 7 8 1
		 * 辽阔 8 10 1
		 */
	}
	/**
	 * HanLPIndexAnalyzer
	 * 这个分词器会对长词进行分割 “中华人民共和国” 会切分成“中华人民共和国” “中华” “人民”等等
	 * @throws IOException
	 */
	@Test
	public void hanLPIndexAnalyzerTest() throws IOException {
		String text = "中华人民共和国很辽阔";
		for (int i = 0; i < text.length(); ++i)
		{
		    System.out.print(text.charAt(i) + "" + i + " ");
		}
		System.out.println();
		Analyzer analyzer = new HanLPIndexAnalyzer();
		TokenStream tokenStream = analyzer.tokenStream("field", text);
		tokenStream.reset();
		while (tokenStream.incrementToken())
		{
		    CharTermAttribute attribute = tokenStream.getAttribute(CharTermAttribute.class);
		    // 偏移量
		    OffsetAttribute offsetAtt = tokenStream.getAttribute(OffsetAttribute.class);
		    // 距离
		    PositionIncrementAttribute positionAttr = tokenStream.getAttribute(PositionIncrementAttribute.class);
		    System.out.println(attribute + " " + offsetAtt.startOffset() + " " + offsetAtt.endOffset() + " " + positionAttr.getPositionIncrement());
		}
		/* 输出：
		 * 中0 华1 人2 民3 共4 和5 国6 很7 辽8 阔9 
		 * 中华人民共和国 0 7 1
		 * 中华人民 0 4 1
		 * 中华 0 2 1
		 * 华人 1 3 1
		 * 人民共和国 2 7 1
		 * 人民 2 4 1
		 * 共和国 4 7 1
		 * 共和 4 6 1
		 * 很 7 8 1
		 * 辽阔 8 10 1

		 */
	}
}