ES借mysql实现词典热更新

这段时间在研究这方面的东西,在此记录下.

翻了很多资料,总结一下,主要是根据下面这篇文章解决的,感谢原作者

原地址: https://blog.csdn.net/wuzhiwei549/article/details/80451302

 

我用的es版本为6.7.1 跟作者有点区别

1.先去git上下载对应的tag版本,这里是6.7.1

https://github.com/medcl/elasticsearch-analysis-ik/tree/v6.7.1

2.导入进idea,pom.xml添加数据库依赖

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.47</version>
        </dependency>

3.在org.wltea.analyzer.dic包下新建HotDictReloadThread类,该类主要是定义一个无线循环的线程,一直调用重新加载词典方法内容如下:

package org.wltea.analyzer.dic;

import org.apache.logging.log4j.Logger;
import org.wltea.analyzer.help.ESPluginLoggerFactory;

public class HotDictReloadThread implements Runnable{
    private static final Logger logger = ESPluginLoggerFactory.getLogger(HotDictReloadThread.class.getName());

    @Override
    public void run() {
        while(true) {
            logger.info("[==========]reload hot dict from mysql......");
            Dictionary.getSingleton().reLoadMainDict();
        }
    }
}

4.在Dictionary.class类中initial方法中添加启动HotDictReloadThread的代码,如下:

/**
	 * 词典初始化 由于IK Analyzer的词典采用Dictionary类的静态方法进行词典初始化
	 * 只有当Dictionary类被实际调用时，才会开始载入词典， 这将延长首次分词操作的时间 该方法提供了一个在应用加载阶段就初始化字典的手段
	 * 
	 * @return Dictionary
	 */
	public static synchronized void initial(Configuration cfg) {
		if (singleton == null) {
			synchronized (Dictionary.class) {
				if (singleton == null) {

					singleton = new Dictionary(cfg);
					singleton.loadMainDict();
					singleton.loadSurnameDict();
					singleton.loadQuantifierDict();
					singleton.loadSuffixDict();
					singleton.loadPrepDict();
					singleton.loadStopWordDict();

                    //启动线程
					new Thread(new HotDictReloadThread()).start();

					if(cfg.isEnableRemoteDict()){
						// 建立监控线程
						for (String location : singleton.getRemoteExtDictionarys()) {
							// 10 秒是初始延迟可以修改的 60是间隔时间 单位秒
							pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
						}
						for (String location : singleton.getRemoteExtStopWordDictionarys()) {
							pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
						}
					}

				}
			}
		}
	}

5.在当前类中添加如下方法:

	private final static  String JDBC_RELOAD_PROP = "jdbc-reload.properties";


	/**
	 * 从mysql加载热更新词典
	 */
	private void loadMySQLExtDict() {
		Connection conn = null;
		Statement stmt = null;
		ResultSet rs = null;

		try {
			Path file = PathUtils.get(getDictRoot(), JDBC_RELOAD_PROP);
			props.load(new FileInputStream(file.toFile()));

			logger.info("[==========]jdbc-reload.properties");
			for(Object key : props.keySet()) {
				logger.info("[==========]" + key + "=" + props.getProperty(String.valueOf(key)));
			}

			logger.info("[==========]query hot dict from mysql, " + props.getProperty("jdbc.reload.sql") + "......");

			conn = DriverManager.getConnection(
					props.getProperty("jdbc.url"),
					props.getProperty("jdbc.user"),
					props.getProperty("jdbc.password"));
			stmt = conn.createStatement();
			rs = stmt.executeQuery(props.getProperty("jdbc.reload.sql"));

			while(rs.next()) {
				String theWord = rs.getString("word");
				logger.info("[==========]hot word from mysql: " + theWord);
				_MainDict.fillSegment(theWord.trim().toCharArray());
			}

			Thread.sleep(Integer.valueOf(String.valueOf(props.get("jdbc.reload.interval"))));
		} catch (Exception e) {
			logger.error("erorr", e);
		} finally {
			if(rs != null) {
				try {
					rs.close();
				} catch (SQLException e) {
					logger.error("error", e);
				}
			}
			if(stmt != null) {
				try {
					stmt.close();
				} catch (SQLException e) {
					logger.error("error", e);
				}
			}
			if(conn != null) {
				try {
					conn.close();
				} catch (SQLException e) {
					logger.error("error", e);
				}
			}
		}
	}

	/**
	 * 从mysql加载停用词
	 */
	private void loadMySQLStopwordDict() {
		Connection conn = null;
		Statement stmt = null;
		ResultSet rs = null;

		try {
			Path file = PathUtils.get(getDictRoot(), JDBC_RELOAD_PROP);
			props.load(new FileInputStream(file.toFile()));

			logger.info("[==========]jdbc-reload.properties");
			for(Object key : props.keySet()) {
				logger.info("[==========]" + key + "=" + props.getProperty(String.valueOf(key)));
			}

			logger.info("[==========]query hot stopword dict from mysql, " + props.getProperty("jdbc.reload.stopword.sql") + "......");

			conn = DriverManager.getConnection(
					props.getProperty("jdbc.url"),
					props.getProperty("jdbc.user"),
					props.getProperty("jdbc.password"));
			stmt = conn.createStatement();
			rs = stmt.executeQuery(props.getProperty("jdbc.reload.stopword.sql"));

			while(rs.next()) {
				String theWord = rs.getString("word");
				logger.info("[==========]hot stopword from mysql: " + theWord);
				_StopWords.fillSegment(theWord.trim().toCharArray());
			}

			Thread.sleep(Integer.valueOf(String.valueOf(props.get("jdbc.reload.interval"))));
		} catch (Exception e) {
			logger.error("erorr", e);
		} finally {
			if(rs != null) {
				try {
					rs.close();
				} catch (SQLException e) {
					logger.error("error", e);
				}
			}
			if(stmt != null) {
				try {
					stmt.close();
				} catch (SQLException e) {
					logger.error("error", e);
				}
			}
			if(conn != null) {
				try {
					conn.close();
				} catch (SQLException e) {
					logger.error("error", e);
				}
			}
		}
	}

6.找到loadMainDict方法,我这边暂未用到停用词典,所以未加,添加下面一句话:

// 从mysql加载词典
this.loadMySQLExtDict();

7.最后就是新建一个数据库配置文件了

jdbc.url=jdbc:mysql://192.35.137.16:3306/test?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=GMT
jdbc.user=admin
jdbc.password=123456
jdbc.reload.sql=select word from hot_words
jdbc.reload.stopword.sql=select stopword as word from hot_stopwords
jdbc.reload.interval=1000

注:这个配置文件只要放到ik目录的config里面就可以了

8.最后就是打包了,clean,package,复制target下的jar 包放入ik目录中即可

9.我的是将mysql-connector-java-5.1.47.jar包放在了：/home/java/jdk1.8.0_121/jre/lib/ext目录下。

10.修改/home/java/jdk1.8.0_121/jre/lib/security/java.policy文件，
在grant下添加如下三行代码：
 

permission java.lang.RuntimePermission "createClassLoader";
permission java.lang.RuntimePermission "getClassLoader";
permission java.net.SocketPermission "192.35.137.16:3306","connect,resolve";

注:192.35.137.16为mysql服务器地址

ok,到这就全部配置完成了,下面就是启动es了