IK分词器的使用

最新推荐文章于 2024-07-19 18:00:00 发布

<黑科技T_T

最新推荐文章于 2024-07-19 18:00:00 发布

阅读量237

点赞数

分类专栏： Java 文章标签： java

本文链接：https://blog.csdn.net/luojiawen208/article/details/119806345

版权

Java 专栏收录该内容

104 篇文章 1 订阅

订阅专栏

<dependency>
  <groupId>com.janeluo</groupId>
  <artifactId>ikanalyzer</artifactId>
  <version>2012_u6</version>
  <exclusions>
    <exclusion>
      <groupId>org.apache.lucene</groupId>
      <artifactId>lucene-core</artifactId>
    </exclusion>
    <exclusion>
      <groupId>org.apache.lucene</groupId>
      <artifactId>lucene-queryparser</artifactId>
    </exclusion>
    <exclusion>
      <groupId>org.apache.lucene</groupId>
      <artifactId>lucene-analyzers-common</artifactId>
    </exclusion>
  </exclusions>
</dependency>

IKAnalyzer.cfg.xml

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
    <comment>IK Analyzer 扩展配置</comment>
    <!--用户可以在这里配置自己的扩展字典 -->
    <entry key="ext_dict">local.dic;</entry>
    <!--用户可以在这里配置自己的扩展停止词字典 -->
    <entry key="ext_stopwords">stop.dic;</entry>
</properties>

local.dic

stop.dic

package org.example.util;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StringUtils;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;

import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

/**
 * @author Gavin.luo
 * @title: IKAnalyzerSupport
 * @projectName MyData
 * @description:
 * @date 2021/7/20 17:07
 */
public class IKAnalyzerSupport {

    private static final Logger LOGGER = LoggerFactory.getLogger(IKAnalyzerSupport.class);

    /**
     * IK分词
     * @param target
     * @return
     */
    public static List<String> iKSegmenterToList(String target) throws Exception {
        if (StringUtils.isEmpty(target)){
            return null;
        }
        List<String> result = new ArrayList<>();
        StringReader sr = new StringReader(target);
        // 关闭智能分词 (对分词的精度影响较大)
        IKSegmenter ik = new IKSegmenter(sr, false);
        Lexeme lex;
        while((lex=ik.next())!=null) {
            String lexemeText = lex.getLexemeText();
            result.add(lexemeText);
        }

        //LOGGER.info("company:{}, iKSegmenterToList:{}", target, JSON.toJSON(result));
        return result;
    }

    public static void main(String[] args) throws Exception {
        String str="http://www.lxtest.cn 是的罗家文同学的个人兴趣分享网站";
        System.out.println(iKSegmenterToList(str));
        List<String> list = iKSegmenterToList(str);
        list.stream().forEach(o->{
            System.out.println(o.hashCode());
        });
    }
}

package org.example.util;


import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;


/**
 * @author Gavin.luo
 * @title: IKAnalyzerUtil
 * @projectName MyData
 * @description:
 * @date 2021/7/20 16:56
 */
public class IKAnalyzerUtil {

    public static List<String> cut(String msg) throws IOException {
        StringReader sr=new StringReader(msg);
        IKSegmenter ik=new IKSegmenter(sr, true);
        Lexeme lex=null;
        List<String> list=new ArrayList<>();
        while((lex=ik.next())!=null){
            list.add(lex.getLexemeText());
        }
        return list;
    }

    public static void main(String[] args) throws IOException {
        String text="http://www.relaxheart.cn 是罗的文同学的个人兴趣分享网站！";
        List<String> list=IKAnalyzerUtil.cut(text);
        System.out.println(list);
    }
}