Lucene7.4做数据匹配

1.jar包引用

        <dependency>
            <groupId>org.apache.lucene</groupId>
            <artifactId>lucene-core</artifactId>
            <version>7.4.0</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.lucene</groupId>
            <artifactId>lucene-queryparser</artifactId>
            <version>7.4.0</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.lucene</groupId>
            <artifactId>lucene-analyzers-common</artifactId>
            <version>7.4.0</version>
            <scope>compile</scope>
        </dependency>
        <dependency>
            <groupId>com.janeluo</groupId>
            <artifactId>ikanalyzer</artifactId>
            <version>2012_u6</version>
        </dependency>

2.重写Ik，兼容高版Lucene7.4

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.synonym.SynonymGraphFilter;
import org.apache.lucene.analysis.synonym.SynonymMap;

import java.io.Reader;
import java.util.List;

public final class IKAnalyzerExt extends Analyzer {
    //同义词库
    private SynonymMap synonymMap;
    //禁用词库
    private List<String> stopWords;
    //默认使用最细力度分词，设置为true表示粗粒度分词
    private boolean useSmart;

    public boolean useSmart() {
        return this.useSmart;
    }

    public void setUseSmart(boolean useSmart) {
        this.useSmart = useSmart;
    }

    public IKAnalyzerExt() {
        //默认使用最细力度分词，设置为true表示粗粒度分词
        this(false);
    }

    public IKAnalyzerExt(boolean useSmart) {
        this.useSmart = useSmart;
    }

    public IKAnalyzerExt(SynonymMap synonymMap, List<String> stopWords, boolean useSmart) {
        //默认使用最细力度分词，设置为true表示粗粒度分词
        this(useSmart);
        this.synonymMap = synonymMap;
        this.stopWords = stopWords;
    }

    @Override
    protected TokenStreamComponents createComponents(String s) {
        Tokenizer _IKTokenizerExt = new IKTokenizerExt(this.useSmart());
        TokenStream tokenStream;

        //同义词初始化
        if (synonymMap != null && synonymMap.fst != null && synonymMap.maxHorizontalContext > 0) {
            tokenStream = new SynonymGraphFilter(_IKTokenizerExt, synonymMap, true);
        } else {
            tokenStream = _IKTokenizerExt;
        }
        //禁用词初始化
        if (stopWords != null && stopWords.size() > 0) {
            tokenStream = new StopFilter(tokenStream, StopFilter.makeStopSet(stopWords));
            return new TokenStreamComponents(_IKTokenizerExt, tokenStream) {
                @Override
                protected void setReader(final Reader reader) {
                    super.setReader(reader);
                }
            };
        }
        return new TokenStreamComponents(_IKTokenizerExt, tokenStream);
    }
}

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.wltea.analyzer.core.IKSegmenter;
import org.wltea.analyzer.core.Lexeme;

import java.io.IOException;

public final class IKTokenizerExt extends Tokenizer {
    //ik分词器
    private IKSegmenter _IKImplement;
    private final CharTermAttribute termAtt = this.addAttribute(CharTermAttribute.class);
    private final OffsetAttribute offsetAtt = this.addAttribute(OffsetAttribute.class);
    private final TypeAttribute typeAtt = this.addAttribute(TypeAttribute.class);
    private int endPosition;

    public IKTokenizerExt(boolean useSmart) {
        this._IKImplement = new IKSegmenter(this.input, useSmart);
    }

    public boolean incrementToken() throws IOException {
        this.clearAttributes();
        Lexeme nextLexeme = this._IKImplement.next();
        if (nextLexeme != null) {
            this.termAtt.append(nextLexeme.getLexemeText());
            this.termAtt.setLength(nextLexeme.getLength());
            this.offsetAtt.setOffset(nextLexeme.getBeginPosition(), nextLexeme.getEndPosition());
            this.endPosition = nextLexeme.getEndPosition();
            this.typeAtt.setType(nextLexeme.getLexemeTypeString());
            return true;
        } else {
            return false;
        }
    }

    public void reset() throws IOException {
        super.reset();
        this._IKImplement.reset(this.input);
    }

    public final void end() {
        int finalOffset = this.correctOffset(this.endPosition);
        this.offsetAtt.setOffset(finalOffset, finalOffset);
    }
}

3、构建匹配引擎

import com.lawtrust.classification.web.modules.task.lucene.ik.IKAnalyzerExt;
import lombok.extern.log4j.Log4j2;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.synonym.SynonymMap;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.*;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.CharsRef;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * @DESCRIPTION:lucene匹配引擎
 * @USER: zenglingceng
 * @DATE: 2023/5/4 13:36
 */
@Log4j2
@Component
public class LuceneEngine {

    /**
     * 构建lucene文档
     * analyzer:分词器
     * features:特征数据
     */
    public IndexSearcher getIndexSearcher(Analyzer analyzer, List<BaseFieldEntity> features) throws IOException {

        //构建lucene文档
        Directory index = new RAMDirectory();
        IndexWriterConfig config = new IndexWriterConfig(analyzer);
        IndexWriter writer = new IndexWriter(index, config);
        /*Field解释
        TextField
        分词，索引，存储可选。
        StringField
        不分词，索引，存储可选。
        IntPoint,LongPoint,FloatPoint,DoublePoint
        分词，索引，不存储(如需存储，添加独立的StoredField)。
        StoredField
        不分词，不索引，存储。
        */
        //fixme 通过数据库获取特征数据构建 lucene文档

        features.stream().forEach(x -> {
            //数据项名称
            String key = x.getFieldClassificationName();
            String description = x.getDescription();
            if(key!=null) {
                //分词
                Document doc = new Document();
                doc.add(new TextField("key", key.toLowerCase(Locale.ROOT), Field.Store.YES));
                doc.add(new StringField("value", x.getId() + "", Field.Store.YES));
                //不分词
                Document doc1 = new Document();
                doc1.add(new StringField("key", key.toLowerCase(Locale.ROOT), Field.Store.YES));
                doc1.add(new StringField("value", x.getId() + "", Field.Store.YES));
                try {
                    writer.addDocument(doc);
                    writer.addDocument(doc1);
                } catch (Exception e) {
                    log.error("特征数据构建lucene文档错误 {}", x);
                }
            }
            //定义说明
            if(description!=null) {
                //分词
                Document doc = new Document();
                doc.add(new TextField("key", description.toLowerCase(Locale.ROOT), Field.Store.YES));
                doc.add(new StringField("value", x.getId() + "", Field.Store.YES));
                //不分词
                Document doc1 = new Document();
                doc1.add(new StringField("key", description.toLowerCase(Locale.ROOT), Field.Store.YES));
                doc1.add(new StringField("value", x.getId() + "", Field.Store.YES));
                try {
                    writer.addDocument(doc);
                    writer.addDocument(doc1);
                } catch (Exception e) {
                    log.error("特征数据构建lucene文档错误 {}", x);
                }
            }
        });
        writer.close();
        IndexReader reader = DirectoryReader.open(index);
        return new IndexSearcher(reader);
    }

    /**
     * 构建分词器
     * thesaurusList:词库
     * useSmart: true表示粗粒度，false表示细粒度
     */
    public IKAnalyzerExt getIkAnalyzerExt(List<BaseThesaurusEntity> thesaurusList, boolean useSmart) throws IOException {

        //构建同义词
        SynonymMap synonymMap = null;
        //构建禁用词
        List<String> stopWords = null;
        //fixme 通过数据库获词库
        if (thesaurusList != null && thesaurusList.size() > 0) {
            //根据wordType分组
            Map<Integer, List<BaseThesaurusEntity>> collect = thesaurusList.stream().collect(Collectors.groupingBy(x -> x.getWordType()));
            //同义词
            List<BaseThesaurusEntity> synonyms = collect.get(0);
            //禁用词
            List<BaseThesaurusEntity> stops = collect.get(1);
            if (synonyms != null && synonyms.size() > 0) {
                SynonymMap.Builder builder = new SynonymMap.Builder(true);
                synonyms.stream().forEach(x -> {
                    String word = x.getWord();
                    String relWord = x.getRelaWord();
                    if (word != null && relWord != null) {
                        String[] split = relWord.split("\\,");
                        for (String synonym : split) {
                            builder.add(new CharsRef(word), new CharsRef(synonym), true);
                        }
                    }
                });
                synonymMap = builder.build();
            }

            if (stops != null && stops.size() > 0) {
                stopWords = new ArrayList<>();
                List<String> finalStopWords = stopWords;
                stops.stream().forEach(x -> {
                    String relWord = x.getRelaWord();
                    if (relWord != null) {
                        String[] split = relWord.split("\\,");
                        for (String stop : split) {
                            finalStopWords.add(stop);
                        }
                    }
                });
            }
        }

        IKAnalyzerExt iKAnalyzerExt = new IKAnalyzerExt(synonymMap, stopWords, useSmart);
        return iKAnalyzerExt;
    }

    public static void main(String[] args) throws IOException, ParseException {
//        LuceneEngine match = new LuceneEngine();
//        IKAnalyzerExt ikAnalyzerExt = match.getIkAnalyzerExt(null,false);
//        String fengci="User_Name";
//        String _fieldName = fengci.replaceAll("_", "");
//        StringReader readerTest = new StringReader(_fieldName);
//        TokenStream tokenStream = ikAnalyzerExt.tokenStream("", readerTest);
//        tokenStream.reset();
//        CharTermAttribute termAttribute = tokenStream.addAttribute(CharTermAttribute.class);
//        while (tokenStream.incrementToken()) {
//            System.out.println(termAttribute.toString());
//        }
//        tokenStream.close();
//        String __fieldName = fengci.replaceAll("_", " ");
//        StringReader readerTest1 = new StringReader(__fieldName);
//        TokenStream tokenStream1 = ikAnalyzerExt.tokenStream("", readerTest1);
//        tokenStream1.reset();
//        CharTermAttribute termAttribute1 = tokenStream1.addAttribute(CharTermAttribute.class);
//        while (tokenStream1.incrementToken()) {
//            System.out.println(termAttribute1.toString());
//        }

//        IndexSearcher searcher = match.getIndexSearcher(ikAnalyzerExt);
//        String query = "w";
//        Query parse = new QueryParser("key", ikAnalyzerExt).parse(query);
//        Query term = new TermQuery(new Term("key", query));
//        Query matchall = new MatchAllDocsQuery();
//        TopDocs docs = searcher.search(parse, 3);
//        System.out.println(docs.totalHits + " matching documents found.");
//        for (ScoreDoc scoreDoc : docs.scoreDocs) {
//            Document doq = searcher.doc(scoreDoc.doc);
//            System.out.println(doq.get("key") + " (score: " + scoreDoc.score + ")");
//            System.out.println(doq.get("value") + " (score: " + scoreDoc.score + ")");
//        }

    }
}