【ES】Elasticsearch实现电商词库提示搜索

Elasticsearch实现电商词库提示搜索

前序

# 自定义拼音分词器
GET _analyze
{
  "text": ["豆腐", "美食", "程序员", "java程序员"],
  "tokenizer": "keyword",
  "filter": [
    {
      "type": "pinyin",
      "keep_first_letter": true,
      "keep_full_pinyin": false,
      "keep_none_chinese": false,
      "keep_separate_first_letter": false,
      "keep_joined_full_pinyin": true,
      "keep_none_chinese_in_joined_full_pinyin": true,
      "none_chinese_pinyin_tokenize": false,
      "limit_first_letter_length": 16,
      "keep_original": true 
    }  
  ]
}

一. 自定义analyzer

1. 数据索引到ES中的时候：例如 “美的电饭锅” -> mddfg, 美的电饭锅, meididianfanguo
2. 在用户搜索的时候，将其中的html标签排除掉，如果是拼音转小写，其他原封不动到ES中搜素。
3. 在数据索引到ES中和搜索的时候使用的 analyzer是 不同的。

# 自定义分词器, lexicon_analyzer是自定义的分词器的名字
PUT lexicon
{
  "settings": {
    "analysis": {
      "analyzer": {
        "lexicon_analyzer" : {
          "char_filter": ["html_strip"],
          "tokenizer": "keyword",
          "filter": [
             "my_lexicon_filter"
          ]
        },
        "lexicon_search_analyzer": {
          "char_filter": ["html_strip"],
          "tokenizer": "keyword",
          "filter": ["lowercase"]
        }
      },
      "filter": {
        "my_lexicon_filter": {
          "type": "pinyin",
          "keep_first_letter": true,
          "keep_full_pinyin": false,
          "keep_none_chinese": false,
          "keep_separate_first_letter": false,
          "keep_joined_full_pinyin": true,
          "keep_none_chinese_in_joined_full_pinyin": true,
          "none_chinese_pinyin_tokenize": false,
          "limit_first_letter_length": 16,
          "keep_original": true
        } 
      }
    }
  }
}

lexicon_analyzer 是数据索引到ES的时候用户的分词器。

lexicon_search_analyzer: 是用户搜索的时候用的分词器。

二. 自定义mappings

# 设置mapping
PUT lexicon/_mapping
{
  "dynamic": false,
  "properties": {
    "words": {
      "type": "completion",
      "analyzer": "lexicon_analyzer",
      "search_analyzer": "lexicon_search_analyzer"
    },
    "id": {
      "type": "long"
    }
  }
}

analyzer是数据索引到ES的时候用的分词器；search_analyzer是用户搜索的时候用的分词器。

三. 数据的测试

3.1 添加测试数据

PUT lexicon/_bulk
{"index": {"_id": 1}}
{"words": "lamer精粹水"}
{"index": {"_id": 2}}
{"words": "lv 女包"}
{"index": {"_id": 3}}
{"words": "macbook pro保护壳"}
{"index": {"_id": 4}}
{"words": "mac口红"}
{"index": {"_id": 4}}
{"words": "jack琼斯"}
{"index": {"_id": 6}}
{"words": "jeep外套男"}
{"index": {"_id": 7}}
{"words": "k20"}

3.2 测试

类型必须是：completion

skip_duplicates: 忽略掉重复。

field: 查询那个字段

GET lexicon/_search
{
  "_source": "id", 
  "suggest": {
    "lexicon_suggest": {
      "prefix": "jackqiongsi",
      "completion": {
         "field": "words",
         "skip_duplicates": true,
         "size": 10
      }
    }
  }
}

3.3 导入数据

创建名称为 es 的数据库，然后执行 lexicon.sql 导入数据

创建 logstash-mysql.conf文件，文件内容如下，放到\logstash-7.4.2\config\目录下

input {
  jdbc {
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://localhost:3306/es?useSSL=false&serverTimezone=UTC"
    jdbc_user => root
    jdbc_password => "root"
    #启用追踪，如果为true，则需要指定tracking_column
    use_column_value => false
    #指定追踪的字段，
    tracking_column => "id"
    #追踪字段的类型，目前只有数字(numeric)和时间类型(timestamp)，默认是数字类型
    tracking_column_type => "numeric"
    #记录最后一次运行的结果
    record_last_run => true
    #上面运行结果的保存位置
    last_run_metadata_path => "mysql-position.txt"
    statement => "SELECT * FROM lexicon"
     #设置对应的时间
    schedule => "0/40 * * * * *"
  }
}
filter {
   
}
output {
  elasticsearch {
    document_id => "%{id}"
    document_type => "_doc"
    index => "lexicon"
    hosts => ["http://localhost:9200"]
  }
  stdout{
    codec => rubydebug
  }
}

将mysql的驱动包放到 logstash家目录的：\logstash-7.4.2\logstash-core\lib\jars 目录下

进到logstash的bin目录下，执行：

logstash.bat -f D:\elasticsearch\logstash-7.4.2\config\logstash-mysql.conf 命令，开始导入数据。

四. Java代码的编写

创建实体类 Lexicon

@Document(indexName = "lexicon", type = "_doc")
public class Lexicon {
    private Long id;
    private String words;

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public String getWords() {
        return words;
    }

    public void setWords(String words) {
        this.words = words;
    }
}

创建 LexiconController

package com.qf.controller;

import com.qf.pojo.Movie;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestBuilder;
import org.elasticsearch.search.suggest.SuggestionBuilder;
import org.elasticsearch.search.suggest.completion.CompletionSuggestion;
import org.elasticsearch.search.suggest.completion.CompletionSuggestionBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.HashSet;
import java.util.List;

@RestController
@RequestMapping("lexicon-suggest")
public class LexiconController {

    private ElasticsearchTemplate elasticsearchTemplate;

    public LexiconController(@Qualifier("elasticsearchTemplate")
                                     ElasticsearchTemplate elasticsearchTemplate) {
        this.elasticsearchTemplate = elasticsearchTemplate;
    }

    @RequestMapping("findAll")
    public Object findAll(String text){

        CompletionSuggestionBuilder completionSuggestionBuilder =
                new CompletionSuggestionBuilder("words")
                        .prefix(text)
                        .skipDuplicates(true);

        SuggestBuilder suggestBuilder =
                new SuggestBuilder().addSuggestion("words_prefix_suggestion", completionSuggestionBuilder);

        //获取Suggest对象
        Suggest suggest = elasticsearchTemplate
                .getClient()
                .prepareSearch("lexicon")//在哪个索引中搜索
                .suggest(suggestBuilder)
                .get()
                .getSuggest();

        Suggest.Suggestion suggestSuggestion = suggest.getSuggestion("words_prefix_suggestion");

        //声明一个集合获取options中的text
        HashSet<String> set = new HashSet<>();

        List entries = suggestSuggestion.getEntries();

        if(entries.size()>0 && entries!=null){
            Object object = entries.get(0);

            if(object instanceof CompletionSuggestion.Entry){

                CompletionSuggestion.Entry entry = (CompletionSuggestion.Entry)object;

                List<CompletionSuggestion.Entry.Option> options = entry.getOptions();

                if(options.size()>0 && options!=null){
                    for(CompletionSuggestion.Entry.Option option : options){
                        set.add(option.getText().toString());
                    }
                }
            }
        }
        return set;
    }
}

在static目录下创建 lexicon.html

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Title</title>
    <script src="js/jquery.min.js"></script>
</head>
<body>
    <input id="autoCompt" type="text" onkeyup="getPrefixResult()"><button>搜索一下</button>
    <div id="callbackValue" style="width: 400px; background-color: antiquewhite;"></div>
</body>
<script>
    function getPrefixResult() {
        // console.log('================')
        var callbackValueBox = $('#callbackValue');
        callbackValueBox.html('');
        var value = $('#autoCompt').val();

        if(value && value.trim() && value.trim().length > 0) {
            $.get("/lexicon-suggest/findAll", {text: value}, function(_data) {
                for(var i = 0; i < _data.length; i++) {
                    callbackValueBox.append('<p>' + _data[i] + "</p>");
                }
            }, "json");
        }
    }
</script>
</html>

五. 在Lexicon搜索的基础上实现商品搜索展示

1.执行goods.sql导入数据到mysql

2.自定义analyzer

# 自定义analyzer
PUT goods
{
  "settings": {
    "analysis": {
      "analyzer": {
        "goods_analyzer": {
          "char_filter": ["html_strip"],
          "tokenizer": "goods_tokenizer"
        }
      },
      "tokenizer": {
        "goods_tokenizer": {
          "type": "hanlp_index",
          "enable_stop_dictionary": true
        }
      }
    }
  }
}

3.定义mapping

PUT goods/_mapping
{
  "properties": {
    "id": {
      "type": "long"
    },
    "no": {
      "type": "long"
    },
    "category": {
      "type": "keyword"
    },
    "title": {
      "type": "text",
      "analyzer": "goods_analyzer"
    },
    "promowords": {
      "type": "text",
      "analyzer": "goods_analyzer",
      "search_analyzer": "goods_analyzer"
    },
    "samllpic": {
      "type": "keyword"
    },
    "price": {
      "type": "double"
    },
    "createtime": {
      "type": "date"
    },
    "stockNum": {
      "type": "long"
    }
  }
}

4.创建logstash-mysql-goods.conf文件，文件内容如下，放到\logstash-7.4.2\config\目录下

input {
  jdbc {
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://localhost:3306/es?useSSL=false&serverTimezone=UTC"
    jdbc_user => root
    jdbc_password => "root"
    #启用追踪，如果为true，则需要指定tracking_column
    use_column_value => false
    #指定追踪的字段，
    tracking_column => "id"
    #追踪字段的类型，目前只有数字(numeric)和时间类型(timestamp)，默认是数字类型
    tracking_column_type => "numeric"
    #记录最后一次运行的结果
    record_last_run => true
    #上面运行结果的保存位置
    last_run_metadata_path => "mysql-goods-position.txt"
    statement => "SELECT i.name category, i.no, g.id, g.title, g.promo_words promowords, g.small_pic samllpic, g.price, g.create_time createtime, g.stock_num stocknum
  from goods g join items i on g.item_id = i.no"
    schedule => "0/40 * * * * *"
  }
}

filter {
   
}

output {
  elasticsearch {
    document_id => "%{id}"
    document_type => "_doc"
    index => "goods"
    hosts => ["http://localhost:9200"]
  }
  stdout{
    codec => rubydebug
  }
}

5.将mysql的驱动包放到 logstash家目录的：\logstash-7.4.2\logstash-core\lib\jars 目录下

6.进到logstash的bin目录下，执行：

logstash.bat -f D:\elasticsearch\logstash-7.4.2\config\logstash-mysql-goods.conf 命令，开始导入数据。

7.创建实体类

package com.qf.pojo;

import org.springframework.data.elasticsearch.annotations.Document;

import java.util.Date;

@Document(indexName = "goods", type = "_doc")
public class Goods {
    private Long id;
    private Long no;
    private String category;
    private String title;
    private String promowords;
    private String samllpic;
    private Double price;
    private String createtime;
    private Long stocknum;

    public Long getId() {
        return id;
    }

    public void setId(Long id) {
        this.id = id;
    }

    public Long getNo() {
        return no;
    }

    public void setNo(Long no) {
        this.no = no;
    }

    public String getCategory() {
        return category;
    }

    public void setCategory(String category) {
        this.category = category;
    }

    public String getTitle() {
        return title;
    }

    public void setTitle(String title) {
        this.title = title;
    }

    public String getPromowords() {
        return promowords;
    }

    public void setPromowords(String promowords) {
        this.promowords = promowords;
    }

    public String getSamllpic() {
        return samllpic;
    }

    public void setSamllpic(String samllpic) {
        this.samllpic = samllpic;
    }

    public Double getPrice() {
        return price;
    }

    public void setPrice(Double price) {
        this.price = price;
    }

    public String getCreatetime() {
        return createtime;
    }

    public void setCreatetime(String createtime) {
        this.createtime = createtime;
    }

    public Long getStocknum() {
        return stocknum;
    }

    public void setStocknum(Long stocknum) {
        this.stocknum = stocknum;
    }
}

8.创建PageInfo

package com.qf.controller;

import java.util.List;

public class PageInfo<T> {
    private List<T> datas;
    private Integer currentPage;

    public List<T> getDatas() {
        return datas;
    }

    public void setDatas(List<T> datas) {
        this.datas = datas;
    }

    public Integer getCurrentPage() {
        return currentPage;
    }

    public void setCurrentPage(Integer currentPage) {
        this.currentPage = currentPage;
    }
}

9.创建controller

package com.qf.controller;

import com.qf.pojo.Goods;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.data.elasticsearch.core.query.SearchQuery;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import java.util.List;

@RestController
@RequestMapping("/goods")
public class GoodsSearchController {

    private ElasticsearchTemplate elasticsearchTemplate;

    public GoodsSearchController(@Qualifier("elasticsearchTemplate")
                                        ElasticsearchTemplate elasticsearchTemplate) {
        this.elasticsearchTemplate = elasticsearchTemplate;
    }

    @GetMapping
    public PageInfo<Goods> indexShow(@RequestParam(defaultValue = "0") Integer currentPage) {
        PageInfo<Goods> pageInfo = new PageInfo<>();

        PageRequest pageRequest = PageRequest.of(currentPage, 10);

        SearchQuery searchQuery = new NativeSearchQueryBuilder()
                .withPageable(pageRequest)
                .withQuery(new MatchAllQueryBuilder())
                .build();

        List<Goods> list = elasticsearchTemplate.queryForList(searchQuery, Goods.class);
        list.forEach(g -> g.setSamllpic("http://localhost/" + g.getSamllpic()));

        pageInfo.setDatas(list);
        pageInfo.setCurrentPage(currentPage);
        return pageInfo;
    }

    @GetMapping("/search")
    public PageInfo<Goods> searchPageGoodsData(String text, @RequestParam(defaultValue = "0") Integer currentPage) {

        PageInfo<Goods> pageInfo = new PageInfo<>();

        PageRequest pageRequest = PageRequest.of(currentPage, 10);

        SearchQuery searchQuery = new NativeSearchQueryBuilder()
                .withIndices("goods")
                .withPageable(pageRequest)
                .withQuery(new MultiMatchQueryBuilder(text, "title", "promowords", "category"))
                .build();

        List<Goods> list = elasticsearchTemplate.queryForList(searchQuery, Goods.class);
        list.forEach(g -> g.setSamllpic("http://localhost/" + g.getSamllpic()));

        pageInfo.setDatas(list);
        pageInfo.setCurrentPage(currentPage);
        return pageInfo;
    }
}

10.引入js以及css文件，创建goods.html页面

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>Title</title>
    <link rel="stylesheet" href="css/bootstrap.min.css">
    <link rel="stylesheet" href="css/jquery-ui.min.css">
    <script src="js/jquery-3.5.0.js"></script>
    <script src="js/jquery-ui.min.js"></script>
    <script src="js/bootstrap.min.js"></script>
    <script src="js/vue.js"></script>
    <script src="js/axios.min.js"></script>
    <style>
        .desc-text {
            height: 50px;
            overflow: hidden;
        }
    </style>
</head>
<body>
<div class="container-fluid">
    <div class="row mt-3 pb-3 mb-3 justify-content-center" style="border-bottom: 1px solid #e2e3e5;">
        <div class="col-10">
            <form class="form-inline justify-content-center" onsubmit="javascript: return false;">
                <div class="form-group col-6 ">
                    <input class="form-control col" id="search-text" onkeyup="searchGood()">
                </div>
                <button type="submit" class="btn btn-primary col-1">搜索一下</button>
            </form>
        </div>
    </div>
</div>
<div class="container" id="app">
    <div class="row row-cols-5 mb-4">
        <div v-for="g in goods" class="col mb-3" :key="g.id">
            <div class="card">
                <img height="196" width="196" alt="暂无图片展示" :src="g.samllpic" class="card-img-top">
                <div class="card-body pb-3">
                    <p class="card-text pb-0 desc-text">{{g.title}} {{g.promowords}}</p>
                    <p class="card-text">￥{{g.price}}</p>
                </div>
            </div>
        </div>
    </div>
</div>
</body>
<script>
    var vm = new Vue({
        el: '#app',
        data() {
            return {
                goods: []
            }
        },
        mounted: function() {
            axios.get('goods')
                .then(res => {
                this.goods = res.data.datas;
        })
        }
    })
    $('#search-text').autocomplete({
        delay: 300,
        max: 20,
        source: function(request, cb) {
            $.ajax({
                url: 'lexicon-suggest/findAll',
                data: {text: request.term},
                type: 'get',
                dataType: 'json',
                success: function(_data) {
                    let tips = [];
                    for(let i = 0; i < _data.length; i++) {
                        tips.push(_data[i]);
                    }
                    cb(tips);
                }
            })
        },
        minlength: 1
    })

    function searchGood() {
            vm.goods = []
            let searchText = $('#search-text').val();
            axios.get('goods/search?text=' + searchText)
                .then(res => {
                for(let i = 0; i < res.data.datas.length; i++) {
                vm.goods.push(res.data.datas[i])
            }
        })
    }
</script>
</html>