玩转Elasticsearch之实战篇

业务背景介绍

公司app的一个小功能，app端提供商户搜索功能，之前功能只是提供一些基础的查询条件，例如商户名称，商户号，手机号码，商户信息，开通时间等，老板看到友商的app上的搜索功能只提供一个文本输入框，可以输入商户名称或者商户号以及其他一些信息，也就是类似全文检索的功能，es的入门研究放在了另一篇玩转Elasticsearch之入门篇

功能介绍&解决方案

具体功能就是app端的一个商户查询功能，一个时间筛选条件，起-止；一个文本搜素框，可以全文模糊搜索，数据量也不算大，就是几张表联合查询，数据存在mysql数据库内，所以搞个单节点就能满足需求
解决方案：
1.数据导入es，首先将需要的所有数据进行表关联查询，将查出的数据导入es中，后续的查询使用es查询
2.数据更新，由于原始数据是存放在mysql中的，数据变动也是直接改变mysql数据，所以需要提供一个数据更新接口，在数据变动后调用数据更新接口更新es中的数据
3.数据查询，使用es的java api进行查询

代码实现

Elasticsearch使用7.3.0版本，项目基于SpringBoot 2.1.5.RELEASE

#pom文件片段，引入es坐标
  <properties>
        <java.version>1.8</java.version>
        <elasticsearch.version>7.3.0</elasticsearch.version>
    </properties>
 <dependencies>
 		 <!--es-->
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>${elasticsearch.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>org.elasticsearch</groupId>
                    <artifactId>elasticsearch</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch</groupId>
            <artifactId>elasticsearch</artifactId>
            <version>${elasticsearch.version}</version>
        </dependency>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>${elasticsearch.version}</version>
        </dependency>
</dependencies>

#yml配置
spring:
  elasticsearch:
    rest:
      uris: 自己esip:9200

1.数据封装model

@Data
@NoArgsConstructor
@AllArgsConstructor
public class MerchantModel {
    @ApiModelProperty(value = "商户ID")
    private Integer id;
    @ApiModelProperty(value = "商户真实姓名")
    private String realName;
    @ApiModelProperty(value = "商户手机号")
    private String merPhone;
    @ApiModelProperty(value = "商户名称")
    private String hardwareType;
    @ApiModelProperty(value = "商户号")
    private String merNo;
    @ApiModelProperty(value = "开通时间")
    private String createTime;
    @ApiModelProperty(value = "代理商编码ID")
    private String agentId;
    @ApiModelProperty(value = "代理商姓名")
    private String agentName;
    @ApiModelProperty(value = "代理商手机号")
    private String agentPhone;
    ....略
}

2.service接口

//商户索引相关操作接口
public interface IMerchantIndexService {
    /**
     * 数据导入es，从mysql查询数据，导入es
     */
    void importAll();

    /**
     * 数据查询
     * @param startTime    开通时间起
     * @param endTime      开通时间止
     * @param keyword      搜索关键字
     * @param current
     * @param size
     * @return
     */
    Page<MerchantModel> searchMerchant(String userId, String scope, String hardwareType, String startTime,
                                       String endTime, String keyword, Integer current, Integer size);
	///创建索引
    void createIndex() throws IOException;
	//删除索引
    void deleteIndex() throws IOException;

    /**
     * 数据更新，局部字段更新，哪些字段变动就传哪个字段
     * @param model
     * @return
     */
    boolean updateIndex(MerchantModel model);
}

3.service接口实现类

@Service
@Slf4j
public class MerchantIndexServiceImpl implements IMerchantIndexService {
    @Resource
    private RestHighLevelClient client;
    @Resource
    private DBHelper dbHelper;


    private static String SQL = "sql就隐藏了，就是几张表联合查询";

    @Override
    public void importAll() {

        try {
            deleteIndex();//先删除索引
            createIndex();//创建索引
        } catch (IOException e) {
            e.printStackTrace();
        }

        BulkProcessor bulkProcessor = getBulkProcessor(client);//BulkProcessor导入数据
        Connection conn = null;
        PreparedStatement ps = null;
        ResultSet rs = null;
        try {
        	//这里使用jdbc从mysql中查询数据，主要是jdbc对批处理支持的好，速度快，也可以用mybatis等框架查询
            conn = dbHelper.getConn();
            ps = conn.prepareStatement(SQL, ResultSet.TYPE_FORWARD_ONLY,
                    ResultSet.CONCUR_READ_ONLY);
            ps.setFetchSize(100);
            rs = ps.executeQuery();
            ResultSetMetaData colData = rs.getMetaData();
            ArrayList<HashMap<String, String>> dataList = new ArrayList<>();

            HashMap<String, String> map = null;
            int count = 0;
            String c = null;
            String v = null;
            while (rs.next()) {
                count++;
                map = new HashMap<>(128);
                for (int i = 1; i <= colData.getColumnCount(); i++) {
                    c = colData.getColumnName(i);
                    v = rs.getString(c);
                    map.put(CaseFormat.LOWER_UNDERSCORE.to(CaseFormat.LOWER_CAMEL, c), v);

                }
                dataList.add(map);
                // 每1万条写一次，不足的批次的最后再一并提交
                if (count % 10000 == 0) {

                    // 将数据添加到 bulkProcessor 中
                    for (HashMap<String, String> hashMap2 : dataList) {
                        bulkProcessor.add(
                                new IndexRequest(MERCHANT_INDEX).source(hashMap2));
                    }
                    // 每提交一次便将map与list清空
                    map.clear();
                    dataList.clear();
                }
            }
            // 处理未提交的数据
            for (HashMap<String, String> hashMap2 : dataList) {
                bulkProcessor.add(
                        new IndexRequest(MERCHANT_INDEX).source(hashMap2));
                System.out.println(hashMap2);
            }
            log.info("-------------------------- 数据导入完成，导入总数:" + count);
            bulkProcessor.flush();
        } catch (Exception e) {
            log.error("导入数据失败", e);
        } finally {
            try {
                rs.close();
                ps.close();
                conn.close();
                bulkProcessor.awaitClose(150L, TimeUnit.SECONDS);

            } catch (Exception e) {
                log.error("关闭资源失败", e);
            }
        }
    }

	//查询
    @Override
    public Page<MerchantModel> searchMerchant(String userId, String scope, String hardwareType,
                                              String startTime, String endTime, String keyword, Integer current, Integer size) {
        if (StringUtils.isEmpty(userId)) {
            return null;
        }
        if (current <= 1) {
            current = 1;
        }
        SearchRequest request = new SearchRequest(MERCHANT_INDEX);
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        //分页
        sourceBuilder.from((current - 1) * size);
        sourceBuilder.size(size);
        //QueryBuilder用来构建查询条件
        QueryBuilder queryBuilder = builderQuery(scope, hardwareType, userId, startTime, endTime, keyword);
        if (queryBuilder == null) {
            return null;
        }
        sourceBuilder.query(queryBuilder);
        sourceBuilder.sort("createTime", SortOrder.DESC);//createTime倒叙排序
        sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        request.source(sourceBuilder);
        List<MerchantModel> records = new ArrayList<>();
        Long total = 0L;
        try {
            SearchResponse response = client.search(request, RequestOptions.DEFAULT);
            SearchHits hits = response.getHits();//查询结果
            TotalHits totalHits = hits.getTotalHits();//总记录数
            total = totalHits.value;
            SearchHit[] searchHits = hits.getHits();//结果数据
            for (SearchHit searchHit : searchHits) {
                //String id = searchHit.getId();//文档ID
                Map<String, Object> sourceAsMap = searchHit.getSourceAsMap();//将查询结果转换为map
                String id = (String) sourceAsMap.getOrDefault("id", 0);
                String merNo = (String) sourceAsMap.getOrDefault("merNo", "");
                String createTime = (String) sourceAsMap.getOrDefault("createTime", "");
                String agentId = (String) sourceAsMap.getOrDefault("agentId", "");
                String agentName = (String) sourceAsMap.getOrDefault("agentName", "");
                String phoneNumber = (String) sourceAsMap.getOrDefault("phoneNumber", "");
                String merPhone = (String) sourceAsMap.getOrDefault("merPhone", "");
                String realName = (String) sourceAsMap.getOrDefault("realName", "");
                String merName = (String) sourceAsMap.getOrDefault("merName", "");
                String chnCode = (String) sourceAsMap.getOrDefault("chnCode", "");
                ........
                ........
                MerchantModel model = new MerchantModel(Integer.parseInt(id), merPhone, merName,  merNo,    createTime, agentId, parentId, realName, phoneNumber,  chnCode, agentName);
                records.add(model);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        Page<MerchantModel> page = new Page<>();//mybatis的分页
        page.setRecords(records);
        page.setTotal(total);
        return page;
    }

    /**
     * 构建es查询条件
     * @param startTime    开通时间起
     * @param endTime      开通时间止
     * @param keyword      搜索关键字
     * @return
     */
    private QueryBuilder builderQuery(String scope, String hardwareType, String userId, String startTime, String endTime, String keyword) {
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        MemInfo memInfo = memInfoService.getById(userId);
        if (StringUtils.isEmpty(scope)) {
            scope = "1";
        }
      //  if ("0".equals(scope)) {
          
      //      boolQueryBuilder.filter(QueryBuilders.prefixQuery("xx", "xxx"));
      //  } 
        if (StringUtils.isNotEmpty(startTime) && StringUtils.isNotEmpty(endTime)) {
            //开通时间范围搜索
            startTime += " 00:00:00";
            endTime += " 23:59:59";
            boolQueryBuilder.must(QueryBuilders.rangeQuery("createTime")
                    .from(startTime)//起始
                    .to(endTime)//结束
                    .includeLower(true)//是否等于起始值
                    .includeUpper(true));//是否等于结束值
        }
        if (StringUtils.isNotEmpty(keyword)) {
        	//搜索关键字模糊搜索
            keyword = "*" + keyword + "*";
            boolQueryBuilder.should(QueryBuilders.wildcardQuery("merName", keyword));//商户名
            boolQueryBuilder.should(QueryBuilders.wildcardQuery("merPhone", keyword));//商户手机号
            ........省略多个字段
            ........
            boolQueryBuilder.minimumShouldMatch(1);//保证shuold条件至少满足一条
        }
        return boolQueryBuilder;
    }

    @Override
    public void createIndex() throws IOException {
        IndicesClient indicesClient = client.indices();//操作索引客户端
        GetIndexRequest getIndexRequest = new GetIndexRequest(MERCHANT_INDEX);
        boolean exists = indicesClient.exists(getIndexRequest, RequestOptions.DEFAULT);//判断索引是否存在
        if (exists) {
            //索引存在
            return;
        }
        String indexMapping = IndexConstants.INDEX_MAPPING;//mapping映射
        CreateIndexRequest createIndexRequest = new CreateIndexRequest(MERCHANT_INDEX);
        createIndexRequest.mapping(indexMapping, XContentType.JSON);
        CreateIndexResponse response = indicesClient.create(createIndexRequest, RequestOptions.DEFAULT);
        boolean acknowledged = response.isAcknowledged();
        if (!acknowledged) {
            throw new RuntimeException("创建索引失败");
        }


    }

    @Override
    public void deleteIndex() throws IOException {
        IndicesClient indicesClient = client.indices();//操作索引客户端
        GetIndexRequest getIndexRequest = new GetIndexRequest(MERCHANT_INDEX);
        boolean exists = indicesClient.exists(getIndexRequest, RequestOptions.DEFAULT);//判断索引是否存在
        if (exists) {
            //索引存在
            DeleteIndexRequest request = new DeleteIndexRequest(MERCHANT_INDEX);
            AcknowledgedResponse delete = indicesClient.delete(request, RequestOptions.DEFAULT);
            if (!delete.isAcknowledged()) {
                throw new RuntimeException("删除索引失败");
            }
        }

    }
	//更新数据
    @Override
    public boolean updateIndex(MerchantModel model) {
        Integer id = model.getId();
        if (id == null) {
            throw new RuntimeException("id不能为空");
        }
        //查询文档是否存在，使用mysql数据库中的id作为条件，查询在es中是否存在该ID的数据
        SearchRequest searchRequest = new SearchRequest(MERCHANT_INDEX);
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.query(QueryBuilders.termQuery("id", id));
        searchRequest.source(sourceBuilder);
        try {
            SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
            SearchHits searchHits = searchResponse.getHits();
            long value = searchHits.getTotalHits().value;
            IndexRequest request = new IndexRequest(MERCHANT_INDEX);
            if (value > 0L) {
                //查询到记录，设置文档id，更新文档
                SearchHit[] hits = searchHits.getHits();
                SearchHit hit = hits[0];
                String docId = hit.getId();
                request.id(docId);//设置文档ID
                Map<String, Object> map = sourceMap(model);
                request.source(map);
                UpdateRequest updateRequest = new UpdateRequest(MERCHANT_INDEX, docId);
                updateRequest.doc(map);
                UpdateResponse updateResponse = client.update(updateRequest, RequestOptions.DEFAULT);
                DocWriteResponse.Result result = updateResponse.getResult();
                System.out.println(result);
            } else {
                //没有记录，插入
                request.source(JsonUtil.objectToJson(model), XContentType.JSON);
                IndexResponse indexResponse = client.index(request, RequestOptions.DEFAULT);
                DocWriteResponse.Result result = indexResponse.getResult();
                System.out.println(result);
            }


        } catch (Exception e) {
            return false;
        }

        return true;
    }
	//反射构建需要更新的字段，不为null的就更新到es
    private Map<String, Object> sourceMap(MerchantModel model) throws Exception {
        Map<String, Object> sourceMap = new HashMap<>();
        Class c = model.getClass();
        Field[] declaredFields = c.getDeclaredFields();

        for (Field field : declaredFields) {
            String fieldName = field.getName();
            String getMethodName = "get" + fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1);
            Method method = c.getMethod(getMethodName);
            Object invoke = method.invoke(model);
            if (invoke != null) {
                sourceMap.put(fieldName, invoke);
            }
        }

        return sourceMap;
    }
	//构建BulkProcessor ，用来导入数据
    private BulkProcessor getBulkProcessor(RestHighLevelClient client) {
        BulkProcessor bulkProcessor = null;
        try {
            BulkProcessor.Listener listener = new BulkProcessor.Listener() {
                @Override
                public void beforeBulk(long executionId, BulkRequest request) {
                    log.info("插入数据条数 : " + request.numberOfActions());
                }

                @Override
                public void afterBulk(long executionId, BulkRequest request,
                                      BulkResponse response) {
                    log.info("************** 成功插入数据条数 : " + request.numberOfActions() + " , id: " + executionId);
                }

                @Override
                public void afterBulk(long executionId, BulkRequest request,
                                      Throwable failure) {
                    log.error("Bulk is unsuccess : " + failure + ",executionId:" + executionId);
                }
            };
            BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer =
                    (request, bulkListener) -> client
                            .bulkAsync(request, RequestOptions.DEFAULT, bulkListener);
            BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer,
                    listener);
            builder.setBulkActions(5000);
            builder.setBulkSize(new ByteSizeValue(100L, ByteSizeUnit.MB));
            builder.setConcurrentRequests(10);
            builder.setFlushInterval(TimeValue.timeValueSeconds(100L));
            builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3));
            // 让参数设置生效
            bulkProcessor = builder.build();
        } catch (Exception e) {
            e.printStackTrace();
            try {
                bulkProcessor.awaitClose(100L, TimeUnit.SECONDS);
            } catch (Exception e1) {
                log.error(e1.getMessage());
            }
        }
        return bulkProcessor;
    }
}

4.常量类&jdbc工具类

//常量
public class IndexConstants {
    public static final String MERCHANT_INDEX = "merchant_index";//索引名称
    //索引映射
    public static final String INDEX_MAPPING = "{\n" +
            "    \"properties\": {\n" +
            "      \"id\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      },\n" +
            "      \"merPhone\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      },\n" +
            "      \"merName\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      },\n" +
            "      \"bpName\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      },\n" +
            "      \"merNo\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      },\n" +
            "      \"createTime\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      },\n" +
            "      \"agentId\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      },\n" +
            "      \"parentId\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      }\n" +
            "      ,\n" +
            "      \"realName\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      }\n" +
            "      ,\n" +
            "      \"phoneNumber\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      }\n" +
            "      ,\n" +
            "      \"longCode\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      }\n" +
            "      ,\n" +
            "      \"chnCode\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      }\n" +
            "      ,\n" +
            "      \"activeSubType\": {\n" +
            "        \"type\": \"keyword\",\n" +
            "        \"store\": true\n" +
            "      }\n" +
            "    }\n" +
            "  }";

}
//DBHelper获取jdbc连接
@Component
public class DBHelper {
    @Value("${spring.datasource.druid.url}")
    private String url;
    @Value("${spring.datasource.druid.driver-class-name}")
    private String driverClassName;
    @Value("${spring.datasource.druid.username}")
    private String userName;
    @Value("${spring.datasource.druid.password}")
    private String password;
    public Connection conn = null;

    public Connection getConn() {
        if (conn != null) {
            return conn;
        }
        try {
            Class.forName(driverClassName);
            conn = DriverManager.getConnection(url, userName, password);//获取连接
        } catch (Exception e) {
            e.printStackTrace();
        }
        return conn;
    }

    public void close() {
        if (conn != null) {
            try {
                conn.close();
            } catch (SQLException throwables) {
                throwables.printStackTrace();
            }
        }
    }
}

总结

以上就是涉及es数据导入，查询，数据更新的相关代码，只能算是一个简单的入门小应用，还远远没有发挥出Elasticsearch真正的威力，先入个门，慢慢积累！！！