一,ES的基本概念
1.什么是全文搜索引擎:
我们搜索时按结构化的拼音搜到读音,然后按其指向的页数,便可找到我们的非结构化数据——也即对字的解释。这种先建立索引,再对索引进行搜索的过程就叫全文检索(Full-text Search)。
代表就是lucence。Lucene是根据关健字来搜索的文本搜索工具,只能在某个网站内部搜索文本内容,不能跨网站搜索。
对lucence进行简化可以采用ES,ES是面向文档的(document oriented)的,对文档(注意不是成行成列的数据)进行索引,搜索,排序,过滤。
对比着传统mysql的概念理解
database对应indexes索引,
table对应types类型,
rows对应doucment文档,
column对应field字段。
创建,删除索引,创建映射,增删改查文档。
查文档分为带分词器的query string查询和不带索引的term查询。
二,前期准备
安装Elasticsearch,头信息,ik分词器。
创建索引库xccourse
创建映射
三,logstash将mysql数据库中的内容同步到ES索引库中的
课程系统远程通过feign调用CMS内容管理系统,并在课程系统中完成发布,并页面保存到本地,,同时将信息保存在coursepub中,完成课程发布。
基于logstash同步mysql数据库中coursepub表的信息到es后台,可以在ES后台“数据浏览”看到与coursepub完全一致的内容。
配置文件mysql.config
1 input {
2 stdin {
3 }
4 jdbc {
5 jdbc_connection_string => "jdbc:mysql://localhost:3306/xc_course?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC"
6 # the user we wish to excute our statement as
7 jdbc_user => "root"
8 jdbc_password => root
9 # the path to our downloaded jdbc driver
10 jdbc_driver_library => "E:/soft/apache-maven-3.3.9/repository/mysql/mysql-connector-java/5.1.36/mysql-connector-java-5.1.36.jar"
11 # the name of the driver class for mysql
12 jdbc_driver_class => "com.mysql.jdbc.Driver"
13 jdbc_paging_enabled => "true"
14 jdbc_page_size => "50000"
15 #要执行的sql文件
16 #statement_filepath => "/conf/course.sql"
17 statement => "select * from course_pub where timestamp > date_add(:sql_last_value,INTERVAL 8 HOUR)"
18 #定时配置
19 schedule => "* * * * *"
20 record_last_run => true
21 last_run_metadata_path => "E:/xuecheng/es/logstash-6.2.1/config/logstash_metadata"
22 }
23 }
24
25
26 output {
27 elasticsearch {
28 #ES的ip地址和端口
29 hosts => "localhost:9200"
30 #hosts => ["localhost:9200","localhost:9202","localhost:9203"]
31 #ES索引库名称
32 index => "xc_course"
33 document_id => "%{id}"
34 document_type => "doc"
35 template =>"E:/xuecheng/es/logstash-6.2.1/config/xc_course_template.json"
36 template_name =>"xc_course"
37 template_overwrite =>"true"
38 }
39 stdout {
40 #日志输出
41 codec => json_lines
42 }
43 }
21行的logstash_metadata文件
--- 2018-06-30 11:26:00.150000000 Z
启动logstash
数据浏览显示mysql的数据。
四,搜索微服务普通分级搜索和按照关键字查询,以及按照分类等级查询,按照分页查询
配置类
ElasticsearchConfig
1 import org.apache.http.HttpHost;
2 import org.elasticsearch.client.RestClient;
3 import org.elasticsearch.client.RestHighLevelClient;
4 import org.springframework.beans.factory.annotation.Value;
5 import org.springframework.context.annotation.Bean;
6 import org.springframework.context.annotation.Configuration;
7
8 /**
9 * @author Administrator
10 * @version 1.0
11 **/
12 @Configuration
13 public class ElasticsearchConfig {
14
15 @Value("${xuecheng.elasticsearch.hostlist}")
16 private String hostlist;
17
18 @Bean
19 public RestHighLevelClient restHighLevelClient(){
20 //解析hostlist配置信息
21 String[] split = hostlist.split(",");
22 //创建HttpHost数组,其中存放es主机和端口的配置信息
23 HttpHost[] httpHostArray = new HttpHost[split.length];
24 for(int i=0;i<split.length;i++){
25 String item = split[i];
26 httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
27 }
28 //创建RestHighLevelClient客户端
29 return new RestHighLevelClient(RestClient.builder(httpHostArray));
30 }
31
32 //项目主要使用RestHighLevelClient,对于低级的客户端暂时不用
33 @Bean
34 public RestClient restClient(){
35 //解析hostlist配置信息
36 String[] split = hostlist.split(",");
37 //创建HttpHost数组,其中存放es主机和端口的配置信息
38 HttpHost[] httpHostArray = new HttpHost[split.length];
39 for(int i=0;i<split.length;i++){
40 String item = split[i];
41 httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
42 }
43 return RestClient.builder(httpHostArray).build();
44 }
45
46 }
api
8 @Api(value = "课程搜索",description = "课程搜索",tags = {"课程搜索"})
9 public interface EsCourseControllerApi {
10
11 @ApiOperation("课程搜索")
12 QueryResponseResult<CoursePub> searchCourse(int page, int size, CourseSearchParam courseSearchParam);
13
14 }
controller
1 @RestController
2 @RequestMapping("/search/course")
3 public class EsCourseController implements EsCourseControllerApi {
4
5 @Autowired
6 private CourseSearchService courseSearchService;
7
8
9 @Override
10 @GetMapping("/list/{page}/{size}")
11 public QueryResponseResult<CoursePub> searchCourse(@PathVariable("page") int page, @PathVariable("size")int size, CourseSearchParam courseSearchParam) {
12 return courseSearchService.searchCourse(page,size,courseSearchParam);
13 }
14 }
yml
1 server:
2 port: ${port:40100}
3 spring:
4 application:
5 name: xc-search-service
6 xc:
7 elasticsearch:
8 hostlist: ${eshostlist:127.0.0.1:9200} #多个结点中间用逗号分隔
9 course:
10 index: xc_course
11 type: doc
service
1 import com.xuecheng.filesystem.framework.domain.course.response.CoursePub;
2 import com.xuecheng.filesystem.framework.domain.search.CourseSearchParam;
3 import com.xuecheng.filesystem.framework.model.response.CommonCode;
4 import com.xuecheng.filesystem.framework.model.response.QueryResponseResult;
5 import com.xuecheng.filesystem.framework.model.response.QueryResult;
6 import org.apache.commons.lang3.StringUtils;
7 import org.elasticsearch.action.search.SearchRequest;
8 import org.elasticsearch.action.search.SearchResponse;
9 import org.elasticsearch.client.RestHighLevelClient;
10 import org.elasticsearch.common.text.Text;
11 import org.elasticsearch.index.query.BoolQueryBuilder;
12 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
13 import org.elasticsearch.index.query.QueryBuilders;
14 import org.elasticsearch.search.SearchHit;
15 import org.elasticsearch.search.SearchHits;
16 import org.elasticsearch.search.builder.SearchSourceBuilder;
17 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
18 import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
19 import org.springframework.beans.factory.annotation.Autowired;
20 import org.springframework.beans.factory.annotation.Value;
21 import org.springframework.stereotype.Service;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.Map;
27
28 @Service
29 public class CourseSearchService {
30
31 @Autowired
32 private RestHighLevelClient restHighLevelClient;
33
34 @Value("${xuecheng.elasticsearch.course.index}")
35 private String index;
36
37 @Value("${xuecheng.elasticsearch.course.type}")
38 private String type;
39
40 /**
41 * 课程搜索
42 * 关键字查询
43 * 一级分类,二级分类,难度等级
44 * 分页查询
45 * 高亮查询
46 * @param page
47 * @param size
48 * @param courseSearchParam
49 * @return
50 */
51 public QueryResponseResult<CoursePub> searchCourse(int page, int size, CourseSearchParam courseSearchParam) {
52
53 SearchRequest searchRequest = new SearchRequest(index);
54 searchRequest.types(type);
55
56 SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
57
58 //boolQueryBuilder must C1 C2 C1 AND C2 or C1 OR C2
59 BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
60
61 //按照关键字查询
62 if (StringUtils.isNotEmpty(courseSearchParam.getKeyword())){
63
64 MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(courseSearchParam.getKeyword(), "name", "description", "teachplan");
65 //查询权重
66 multiMatchQueryBuilder.field("name",10);
67 multiMatchQueryBuilder.minimumShouldMatch("70%");
68
69 boolQueryBuilder.must(multiMatchQueryBuilder);
70 }
71
72 //过滤查询&等值查询
73 //一级分类查询
74 if (StringUtils.isNotEmpty(courseSearchParam.getMt())){
75 boolQueryBuilder.filter(QueryBuilders.termQuery("mt",courseSearchParam.getMt()));
76 }
77
78 //二级分类查询
79 if (StringUtils.isNotEmpty(courseSearchParam.getSt())){
80 boolQueryBuilder.filter(QueryBuilders.termQuery("st",courseSearchParam.getSt()));
81 }
82
83 //难度等级查询
84 if (StringUtils.isNotEmpty(courseSearchParam.getGrade())){
85 boolQueryBuilder.filter(QueryBuilders.termQuery("grade",courseSearchParam.getGrade()));
86 }
87
88 //分页查询
89 if (page <=0){ //当前页
90 page =1;
91 }
92 if (size <= 0 ){
93 size = 10;
94 }
95
96 int start = (page-1)*size;
97 //从哪开始查
98 sourceBuilder.from(start);
99 //每页显示多少
100 sourceBuilder.size(size);
101
102 //高亮查询
103 HighlightBuilder highlightBuilder = new HighlightBuilder();
104 //高亮前缀
105 highlightBuilder.preTags("<font class='eslight'>");
106 //高亮后缀
107 highlightBuilder.postTags("</font>");
108 //高亮域
109 highlightBuilder.fields().add(new HighlightBuilder.Field("name"));
110 sourceBuilder.highlighter(highlightBuilder);
111
112
113 sourceBuilder.query(boolQueryBuilder);
114
115 searchRequest.source(sourceBuilder);
116
117 SearchResponse searchResponse = null;
118 try {
119 searchResponse = restHighLevelClient.search(searchRequest);
120 } catch (IOException e) {
121 e.printStackTrace();
122 }
123 //获取查询结果
124 SearchHits hits = searchResponse.getHits();
125 SearchHit[] searchHits = hits.getHits();
126
127 List<CoursePub> coursePubList = new ArrayList<>();
128
129 for (SearchHit searchHit : searchHits) {
130
131 Map<String, Object> sourceAsMap = searchHit.getSourceAsMap();
132 CoursePub coursePub = new CoursePub();
133
134 String id = (String) sourceAsMap.get("id");
135 coursePub.setId(id);
136
137 //名称
138 String name = (String) sourceAsMap.get("name");
139 //获取高亮
140 Map<String, HighlightField> highlightFields = searchHit.getHighlightFields();
141 if (highlightFields != null){
142 HighlightField highlightField = highlightFields.get("name");
143 if (highlightField != null){
144 Text[] fragments = highlightField.fragments();
145 if (fragments !=null){
146 StringBuffer stringBuffer = new StringBuffer();
147 for (Text fragment : fragments) {
148 stringBuffer.append(fragment);
149 }
150 name = stringBuffer.toString();
151 }
152 }
153 coursePub.setName(name);
154 }
155
156
157 //图片
158 String pic = (String) sourceAsMap.get("pic");
159 coursePub.setPic(pic);
160
161 //价格
162 Float price = null;
163 if (sourceAsMap.get("price") != null){
164 price = Float.parseFloat(String.valueOf(sourceAsMap.get("price")));
165 }
166 coursePub.setPrice(price);
167
168 //原价
169 Float price_old = null;
170 if (sourceAsMap.get("price_old")!=null){
171 price_old = Float.parseFloat(String.valueOf(sourceAsMap.get("price_old")));
172 }
173 coursePub.setPrice_old(price_old);
174
175 coursePubList.add(coursePub);
176
177 }
178
179 //数据封装
180 QueryResult queryResult = new QueryResult();
181 queryResult.setTotal(hits.getTotalHits());
182 queryResult.setList(coursePubList);
183 return new QueryResponseResult<CoursePub>(CommonCode.SUCCESS,queryResult);
184 }
185 }
nginx代理转发,用户请求/course/search的nginx将请求转发给nuxt。js服务。nginx在转发时根据每台nuxt服务器的负载情况进行转发,实现负载均衡
最后实现走进‘’‘课程搜索’可以进行关键字查询并对关键字实现高亮显示,并采用二级联动实现分级。