记录阿里云Elastic Search实例使用经验

阿里云elastic search的简单使用步骤

购买服务器

这里就不详细写如何购买服务器了(需要有elastic search实例和logstash实例)

购买完成开始设置并使用

1.打开实例管理页面

2.设置下kibana语言方便操作

3.登录kibana可视化管理平台

默认账号 elastic 密码为创建实例是设置的es密码
登录kibana控制台找到开发工具（图标：小扳手），执行下列操作
#创建索引 自定义索引名称test_index
#设置分片数为5 副本数为1
#设置映射 ik_max_word为细分词 ik_smart为粗略分词 type字段类型
#type 对应

type	mysql
keyword(不清楚就用keyword)	int,byte,long
varchar	text
float	浮点数或者Bigdecimal
date	date

执行创建索引

示例:

#创建索引
PUT /test_index  #索引名称
{
  "settings": {
    "number_of_shards": 5, #分片数
    "number_of_replicas": 1  #副本数
  },
  "mappings": {
    "properties" : {  
			"ProId" : { 
               "type" : "keyword" 
            },
            "ProName" : { 
               "analyzer" : "ik_max_word",
               "search_analyzer" : "ik_smart",
               "type" : "text"
            },
			"Unit" : {
               "type" : "text"
            },
			"GiveIntegral" : {
               "type" : "float"
            },
			"ProImg" : {
               "index" : false,
               "type" : "keyword"
            },
      "AddDate" : {
               "format" : "yyyy-MM-dd HH:mm:ss",
               "type" : "date"
            },
      "categoryId" : {
               "type" : "keyword"
            }
         }
      }
  }
#设置为非只读模式（如需删除操作）
PUT /_settings
{
  "index": {
    "blocks": {
      "read_only_allow_delete": "false"
    }
  }
}

#获取索引所有数据及其配置
GET /test_index/_search

#根据id删除索引中的数据
DELETE /test_index/_doc/id值

#删除查询到的数据
POST /test_index/_doc/_delete_by_query
{
  "query": {
    "match_all": {}
  }
}

#此处为测试英文分词
POST /test_index/_analyze
{
  "analyzer" : "ik_max_word",
  "text": "where are you"
}
#此处为测试中文分词
#粗细分词自己设定
POST /test_index/_analyze
{
  "analyzer" : "ik_smart",
  "text": "中华人民共和国"
}

进入索引管理查看索引创建是否成功

接下来就可以同步数据到索引了

打开logstash实例

上传所需要的mysql 驱动jar包

这里mysql驱动jar包可以从自己maven仓库中取出来上传上去

创建管道

input {
  jdbc {
    jdbc_driver_class => "com.mysql.jdbc.Driver"
    jdbc_driver_library => "/ssd/1/share/ls-cn-替换你的实例地址/logstash/current/config/custom/mysql-connector-java-8.0.18.jar"
    jdbc_connection_string => "jdbc:mysql://xxxx:3306/数据库名称?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowLoadLocalInfile=false&autoDeserialize=false&serverTimezone=CTT&tinyInt1isBit=false&zeroDateTimeBehaviro=convertToNull"
    jdbc_user => "xxx"
    jdbc_password => "xxx"
    jdbc_paging_enabled => "true"
    jdbc_page_size => "50000"
    statement => "sql语句一定要加此条件时间字段自定义 where 新增时间字段 >= :sql_last_value OR 修改时间字段 >= :sql_last_value"
    schedule => "* * * * *" 
    record_last_run => true
    last_run_metadata_path => "/ssd/1/ls-cn-替换你的实例地址/logstash/data/last_run_metadata_update_time.txt"
    clean_run => false
    # 设置为true时，sql_last_value的值是tracking_column的值；设置为false是，sql_last_value的值是上次执行的值。
    use_column_value => false
  }
}
filter {
}
output {
 elasticsearch {
    hosts => "es-cn-xxxx.elasticsearch.aliyuncs.com:9200"
    index => "索引名称"
    user => "elastic"
    document_id => "%{文档id可以是主键id}"
    password => "es访问密码"
  }
}

讲上边配置对应填进去

jdbc_driver_library 对应jdbc文件路径

登录kibana可视化查看索引同步状态(文档数和大小是否变化)

最后对应java搜索代码也贴一下

对应依赖

  	<dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>6.5.4</version>
    </dependency>
    <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>6.5.4</version>
    </dependency>

yml配置

es:
  elasticsearch:
#    hostlist: es-cn-xxxx.public.elasticsearch.aliyuncs.com #本地测试公网(公网需要加白名单)
    hostlist: es-cn-xxxx.elasticsearch.aliyuncs.com #线上用内网
    user: elastic
    password: es访问密码
  genteral_store:
    index: 索引名称
    #source_field: 这里是索引中的字段多个,号分割,与索引创建的字段名称对应
    source_field: proid,vipprice,prosum等

config配置类

import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
 * @author Administrator
 * @version 1.0
 **/
@Configuration
public class ElasticsearchConfig {

    @Value("${es.elasticsearch.hostlist}")
    private String hostlist;
    @Value("${es.elasticsearch.user}")
    private String user;
    @Value("${es.elasticsearch.password}")
    private String password;

    /**
     * Java High Level REST Client（本章节以此为例）：Elasticsearch Client官方高级客户端。基于低级客户端
     * 同步调用方法立即返回一个Response对象。
     * 而异步调用方法（方法名以async结尾）依赖于监听，当有请求返回或是错误返回时，该监听会通知到对应的方法继续执行。
     */
//    @Bean
//    public RestHighLevelClient restHighLevelClient(){
//        //解析hostlist配置信息
//        String[] split = hostlist.split(",");
//        //创建HttpHost数组，其中存放es主机和端口的配置信息
//        HttpHost[] httpHostArray = new HttpHost[split.length];
//        for(int i=0;i<split.length;i++){
//            String item = split[i];
//            httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
//        }
//        //创建RestHighLevelClient客户端
//        return new RestHighLevelClient(RestClient.builder(httpHostArray));
//    }

    //项目主要使用RestHighLevelClient，对于低级的客户端暂时不用
//    @Bean
//    public RestClient restClient(){
//        //解析hostlist配置信息
//        String[] split = hostlist.split(",");
//        //创建HttpHost数组，其中存放es主机和端口的配置信息
//        HttpHost[] httpHostArray = new HttpHost[split.length];
//        for(int i=0;i<split.length;i++){
//            String item = split[i];
//            httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
//        }
//        return RestClient.builder(httpHostArray).build();
//    }


    /**
     * Java High Level REST Client（本章节以此为例）：Elasticsearch Client官方高级客户端。基于低级客户端
     * 同步调用方法立即返回一个Response对象。
     * 而异步调用方法（方法名以async结尾）依赖于监听，当有请求返回或是错误返回时，该监听会通知到对应的方法继续执行。
     */
    //阿里云高级客户端配置
    @Bean
    public RestHighLevelClient restHighLevelClient() {

        // 阿里云ES集群需要basic auth验证。
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        //访问用户名和密码为您创建阿里云Elasticsearch实例时设置的用户名和密码，也是Kibana控制台的登录用户名和密码。
        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(user, password));

        // 通过builder创建rest client，配置http client的HttpClientConfigCallback。
        // 单击所创建的Elasticsearch实例ID，在基本信息页面获取公网地址，即为ES集群地址。
        RestClientBuilder builder = RestClient.builder(new HttpHost(hostlist, 9200))
                .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                    @Override
                    public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                    }
                });

        //创建RestHighLevelClient客户端
        return new RestHighLevelClient(builder);
    }

    //项目主要使用RestHighLevelClient，对于低级的客户端暂时不用
    /**
     * Java Low Level REST Client：Elasticsearch Client低级别客户端。它允许通过HTTP请求与ES集群进行通信。
     * API本身不负责数据的编码解码，由用户去编码解码。它与所有的ES版本兼容。
     * @return
     */
    @Bean
    public RestClient restClient() {
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(user, password));
        RestClient restClient = RestClient.builder(new HttpHost(hostlist, 9200))
                .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
                    @Override
                    public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
                        return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
                    }
                }).build();
        return restClient;
    }
}

model层

import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;

import java.io.Serializable;

/**
 * 搜索请求参数
 */
@Data
@NoArgsConstructor
@ToString
public class GeneralStoreSearchParam{
    Integer page;//页码

    String keyword; //关键词

    Integer sort;//排序方式

    String cid; //分类id
    String cidtwo; //分类id
    String cidthree; //分类id

    Boolean min_dis;//从小到大

    Boolean max_dis;//从大到小

    Double price_max;//价格最高区间

    Double price_min;//价格最低区间

    String from_where;//商品来源

    String jieri_id;

    String shop_type; //商品类型
}

controller层

import com.zzfeidu.model.GeneralStoreSearchParam;
import com.zzfeidu.model.QueryPageResult;
import com.zzfeidu.service.ElasticSearchService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;

/**
 * 搜索服务
 */
@CrossOrigin(origins = "*", maxAge = 3600)
@RestController
@RequestMapping("/search")
public class ElasticSearchController{

    @Autowired
    private ElasticSearchService elasticSearchService;

    /**
     * 根据条件全文检索
     * @param generalStoreSearchParam
     * @return
     */
    @GetMapping("/list")
    public QueryPageResult search(GeneralStoreSearchParam generalStoreSearchParam) {
        return elasticSearchService.getlist(generalStoreSearchParam);
    }
}

service实现层

import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;

import java.io.IOException;
import java.math.BigDecimal;
import java.util.*;

/**
 * 搜索服务
 */
@Service
@Slf4j
public class ElasticSearchService {

    @Autowired
    RestClient restClient;
    @Autowired
    RestHighLevelClient restHighLevelClient;
    @Value("${es.genteral_store.index}")
    String index;
//    @Value("${es.genteral_store.type}")
//    String doc;
    @Value("${es.genteral_store.source_field}")
    String source_field;
    @Autowired
    SupplierMapper supplierMapper;
    @Autowired
    ProductMapper productMapper;

    /**
     * 根据条件进行搜索相关商品
     * */
    public QueryPageResult<Productdetail> getlist(GeneralStoreSearchParam param) {
        //防止空指针
        if(param == null){
            param = new GeneralStoreSearchParam();
        }
        log.info("----进入搜索参数--param="+param);
        //定义参数集queryResult
        QueryPageResult<Productdetail> queryResult = new QueryPageResult<>();
        List<Productdetail> list = new ArrayList<>();
        log.info("----高级搜索-----");
        //获取搜索请求对象
        SearchRequest general_store = new SearchRequest(index);
        //指定类型
//        general_store.types(doc);
        //搜索源构建对象
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        //设置源字段过滤，第一个参数结果集包括那些字段，第二个参数标识结果集不包括哪些字段
        String[] source_field_array = source_field.split(",");
        searchSourceBuilder.fetchSource(source_field_array,new String[]{});

        //设置分页参数
        if(param.getPage() == null || param.getPage() <= 0){
            param.setPage(1);
        }
        int page = (param.getPage() - 1) * 10;
        searchSourceBuilder.from(page);//当前页
        searchSourceBuilder.size(10);//当前页

        //创建布尔搜索对象
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();

        //multiMatchQuery
        if(StringUtils.isNotEmpty(param.getKeyword())){
            //首先定义一个multiMatch搜索
            MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(param.getKeyword(), "proname", "description","keywords","categoryname1","categoryname2","categoryname3")
                    .minimumShouldMatch("100%")
                    .field("kedwords", 10)
                    .field("proname",8)
                    .field("categoryname1",8)
                    .field("categoryname2",8)
                    .field("categoryname3",8);
            //添加进布尔搜索
            boolQueryBuilder.must(multiMatchQueryBuilder);
        }
        //分类过滤
        if(StringUtils.isNotEmpty(param.getCid())){
            //首先定义一个termQuery精确查询
            TermQueryBuilder termQuery = QueryBuilders.termQuery("categoryid", param.getCid());
            //添加进布尔搜索
            boolQueryBuilder.must(termQuery);
        }
        //分类过滤
        if(StringUtils.isNotEmpty(param.getCidtwo())){
            //首先定义一个termQuery精确查询
            TermQueryBuilder termQuery = QueryBuilders.termQuery("categorycode", param.getCidtwo());
            //添加进布尔搜索
            boolQueryBuilder.must(termQuery);
        }
        //分类过滤
        if(StringUtils.isNotEmpty(param.getCidthree())){
            //首先定义一个termQuery精确查询
            TermQueryBuilder termQuery = QueryBuilders.termQuery("categorythird", param.getCidthree());
            //添加进布尔搜索
            boolQueryBuilder.must(termQuery);
        }
        //商品类型过滤
        if(StringUtils.isNotEmpty(param.getShop_type())){
            //首先定义一个termQuery精确查询
            TermQueryBuilder termQuery = QueryBuilders.termQuery("shop_type", param.getShop_type());
            //添加进布尔搜索
            boolQueryBuilder.must(termQuery);
        }
        TermQueryBuilder termQuery = QueryBuilders.termQuery("isonsell", 1);
        boolQueryBuilder.must(termQuery);
        //添加过虑器
        //价格区间过虑
        if(param.getPrice_min() != null && param.getPrice_max() != null){
            //设置最高价 - 最低价
            boolQueryBuilder.filter(QueryBuilders.rangeQuery("balanceprice").gte(param.getPrice_min()).lte(param.getPrice_max()));
        }else {
            //设置 最低价 - xxx
            if(param.getPrice_min() != null){
                boolQueryBuilder.filter(QueryBuilders.rangeQuery("balanceprice").gte(param.getPrice_min()));
            }
            //设置 0 - 最高价
            if(param.getPrice_max() != null){
                boolQueryBuilder.filter(QueryBuilders.rangeQuery("balanceprice").gte(0).lte(param.getPrice_max()));
            }
        }

        //获取排序方式
        Integer sort = param.getSort();

        //设置销量降序
        if(sort != null){
            if(sort == 1){
                searchSourceBuilder.sort("sort", SortOrder.ASC);
            }
            if(sort == 2){
                searchSourceBuilder.sort("prosum", SortOrder.DESC);
            }
            //设置销量升序
            if(sort == 3){
                searchSourceBuilder.sort("prosum", SortOrder.ASC);
            }
            //设置价格升序
            if(sort == 4){
                searchSourceBuilder.sort("balanceprice", SortOrder.ASC);
            }
            //设置价格降序
            if(sort == 5){
                searchSourceBuilder.sort("balanceprice", SortOrder.DESC);
            }
            //链分值降序
            if(sort == 6){
                searchSourceBuilder.sort("lfz", SortOrder.DESC);
            }
            //链分值升序
            if(sort == 7){
                searchSourceBuilder.sort("lfz", SortOrder.ASC);
            }
            //链分值比例降序
            if(sort == 8){
                searchSourceBuilder.sort("lfzbili", SortOrder.DESC);
            }
            //链分值比例升序
            if(sort == 9){
                searchSourceBuilder.sort("lfzbili", SortOrder.ASC);
            }
        }
        //设置高亮显示
//        highlightBuilder.preTags("<span style=\"color:red\">");
//        highlightBuilder.postTags("</span>");
        //定义高亮--
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.preTags("<text style=\"color: #ffac34;\">");
        highlightBuilder.postTags("</text>");
        highlightBuilder.fields().add(new HighlightBuilder.Field("proname"));
        searchSourceBuilder.highlighter(highlightBuilder);
        searchSourceBuilder.query(boolQueryBuilder);
        //向搜索请求对象中设置搜索源
        general_store.source(searchSourceBuilder);
        //执行搜索，向es发起http请求
        SearchResponse searchResponse = null;
        try {
            searchResponse = restHighLevelClient.search(general_store);
        } catch (IOException e) {
            e.printStackTrace();
        }

        //搜索结果
        SearchHits hits = searchResponse.getHits();
        //设置总纪录数
        queryResult.setAll_count(hits.getTotalHits() + "");
        //匹配度较高的前n个文档
        SearchHit[] searchHits = hits.getHits();
        for(SearchHit hit : searchHits){



            Productdetail productdetail = new Productdetail();
            String id = hit.getId();//id
            //源文档内容
            Map<String, Object> sourceAsMap = hit.getSourceAsMap();
            String ProId = sourceAsMap.get("proid").toString();//商品id
            String ProName = (String) sourceAsMap.get("proname");//商品名称


            //取出高亮字段--day12
            Map<String, HighlightField> highlightFields = hit.getHighlightFields();
            if(highlightFields.get("proname")!=null){
                HighlightField highlightField = highlightFields.get("proname");
                Text[] fragments = highlightField.fragments();
                StringBuffer stringBuffer = new StringBuffer();
                for(Text text:fragments){
                    stringBuffer.append(text);
                }
                ProName = stringBuffer.toString();
                ProName = "<text>" + ProName + "</text>";
            }
            System.out.println("高亮后的ProName："+ProName);


            String qiqiuproimgpath = (String) sourceAsMap.get("qiqiuproimgpath");//商品小图片路径
            String shop_type = sourceAsMap.get("shop_type").toString(); //商家信息
            String VipPrice =  sourceAsMap.get("vipprice").toString(); //代理价
            String MarketPrice =  sourceAsMap.get("marketprice").toString(); //市场价
            String BalancePrice = sourceAsMap.get("balanceprice").toString(); //结算价
            String IsHit = sourceAsMap.get("ishit").toString(); //商品所属区域
            String ConsumeIntegral = sourceAsMap.get("consumeintegral").toString(); //购买此商品消耗的积分
            String ProImg = (String) sourceAsMap.get("proimg"); //商品图片路径
            String prosum = sourceAsMap.get("prosum").toString(); //销量
            String categoryId = sourceAsMap.get("categoryid").toString(); //分类id1
            String categoryCode = sourceAsMap.get("categorycode").toString(); //分类id2
            String categoryThird= sourceAsMap.get("categorythird").toString(); //分类id3
            String SupplierId = sourceAsMap.get("supplierid").toString(); //商家信息
            String lfz = "0";
            if(sourceAsMap.get("lfz") != null){
                lfz = sourceAsMap.get("lfz").toString(); //商家信息
            }
            String lfzbili = sourceAsMap.get("lfzbili").toString(); //商家信息
            String suppliername = sourceAsMap.get("suppliername").toString(); //商家信息

            productdetail.setId(id);
            productdetail.setProname(ProName);//商品名称
            productdetail.setProid(ProId);//商品id
            productdetail.setCategoryId(categoryId);//分类id
            productdetail.setConsumeintegral(ConsumeIntegral);//购买此商品消耗的积分
            productdetail.setMarketprice(MarketPrice);//市场价
            productdetail.setWholesale_price(MarketPrice);//市场价
            if (StringUtils.isNotEmpty(shop_type) && Integer.parseInt(shop_type) != 4) {
                double vip = StringUtils.isEmpty(VipPrice) ? 0 : Double.parseDouble(VipPrice) * 0.2;
                productdetail.setVipprice(new BigDecimal(vip).setScale(2,BigDecimal.ROUND_HALF_UP).toString());
            }else {
                productdetail.setVipprice(VipPrice);//代理价
            }
            productdetail.setPronum(prosum);//销量
            productdetail.setIshit(IsHit);//商品所属区域
            productdetail.setLf(lfz);//商品所属区域
            productdetail.setLfzbili(lfzbili);//商品所属区域
            productdetail.setShop_type(shop_type);//商品所属区域
            productdetail.setCategoryCode(categoryCode);//商品所属区域
            productdetail.setCategoryThird(categoryThird);//商品所属区域
            productdetail.setSuppliername(suppliername);//商品所属区域
            if (!ProImg.startsWith("http")) {
                productdetail.setProimg(MyConfig.utl + ProImg);//商品图片路径
            } else {
                productdetail.setProimg(ProImg);//商品图片路径
            }

            productdetail.setSupplierid(SupplierId);//商家id
            productdetail.setWholesale_price(BalancePrice);//批发价
            productdetail.setQiqiuproimgpath(qiqiuproimgpath);//商品小图片路径

            list.add(productdetail);
        }
//            }
//        queryResult.setData_list(list);

        HashMap<String, List<Productdetail>> map = new HashMap<>();
        map.put("data_list",list);
        queryResult.setData(map);
        queryResult.setMsg("查询成功！");
        queryResult.setStatus("1");
        log.info("----搜索结果 queryResult="+queryResult);

        return queryResult;
    }
}

参考链接

附kibana使用文档:https://www.elastic.co/guide/en/kibana/current/index.html
附es映射参考路径:https://blog.csdn.net/u013545439/article/details/102799518