关键代码
常规操作
ElasticSearchClientConfig
package com.ge.es.hcapmelsservice.config;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.env.Environment;
@Configuration
public class ElasticSearchClientConfig {
@Autowired
private Environment env;
@Bean
public RestHighLevelClient restHighLevelClient(){
String url = env.getProperty("web-context.esUrl");
RestHighLevelClient client = new RestHighLevelClient(
RestClient.builder(
new HttpHost("127.0.0.1", 9201, "http")));
return client;
}
}
SearchController
package com.ge.es.hcapmelsservice.controller;
import com.ge.es.hcapmelsservice.pojo.CmsContent;
import com.ge.es.hcapmelsservice.pojo.UserProfile;
import com.ge.es.hcapmelsservice.service.SearchService;
import com.ge.es.hcapmelsservice.service.UserProfileService;
import com.ge.es.hcapmelsservice.util.BizStatusCode;
import com.ge.es.hcapmelsservice.util.ResponseData;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@Api("SearchController")
@RestController
@RequestMapping("/api/apm/es/search/")
public class SearchController {
private static Logger log = LoggerFactory.getLogger(com.ge.es.hcapmelsservice.controller.SearchController.class);
@Autowired
@Qualifier("restHighLevelClient")
private RestHighLevelClient client;
@Autowired
private UserProfileService userProfileService;
@Autowired
private SearchService searchService;
private ResponseEntity<ResponseData> returnDataResponseEntity(Object msg) {
return ResponseEntity.ok().body(
new ResponseData(
BizStatusCode.OK,
"The EsController was created successfully!",
null,
msg
)
);
}
@RequestMapping(value = "/getSearch", method = RequestMethod.POST, produces = MediaType.APPLICATION_JSON_VALUE)
@ApiOperation(value = "通过搜索内容获取文档信息")
public ResponseEntity<ResponseData> search3(
@RequestHeader(value = "Authorization",required = true) String authorization,
@RequestParam(value = "content",required = false) String content,
@RequestParam(value = "articleType",required = false) String articleType,
@RequestParam(value = "modality",required = false) String modality,
@RequestParam(value = "channelId",required = false) String channelId,
@RequestParam(value = "channelId2",required = false) String channelId2,
@RequestParam(value = "articleTypeFlag",required = false) String articleTypeFlag,//articleTypeFlag 1: 当articleType为0时 返回每组两条的所有数据 articleTypeFlag=0:当 当articleType为0时 返回不分组的数
@RequestParam(value = "pageNum",required = false) Integer pageNum,
@RequestParam(value = "ids",required = false) List<Integer> ids
) throws IOException {
UserProfile user = userProfileService.getUserProfileByLoginName(authorization);
if (user == null) {
log.info("insertApproveTable error:用户未登陆");
return returnDataResponseEntity( "用户未登陆!");
}
int userId = user.getUserAccount().getId();
String loginName = user.getUserAccount().getLoginName();
Map<String, Object> map = searchService.getSearchData(
null==content?"":content,
null==channelId?"0":channelId,
null==articleType?"0":articleType,
userId,
null==pageNum?0:pageNum,
null==channelId2?"0":channelId2,
null==articleTypeFlag?"0":articleTypeFlag,
loginName,
modality,
ids);
return returnDataResponseEntity(map);
}
@ResponseBody
@RequestMapping(method = RequestMethod.DELETE)
@ApiOperation(value = "deleteSlidesImg")
public ResponseEntity<ResponseData> deleteSlidesImg(
@RequestHeader(value = "Authorization") String authorization
) throws IOException {
UserProfile user = userProfileService.getUserProfileByLoginName(authorization);
if (user == null) {
log.info("insertApproveTable error:用户未登陆");
return returnDataResponseEntity( "用户未登陆!");
}
int userId = user.getUserAccount().getId();
Map<String, Object> map = searchService.testDeleteIndex();
return returnDataResponseEntity(map);
}
@ResponseBody
@RequestMapping(value = "/bulkUpdate",method = RequestMethod.POST)
@ApiOperation(value = "批量更新")
public ResponseEntity<ResponseData> BulkRequestUpdate(
@RequestHeader(value = "Authorization") String authorization,
@RequestBody List<CmsContent> cmsContents
) throws IOException {
UserProfile user = userProfileService.getUserProfileByLoginName(authorization);
if (user == null) {
log.info("BulkRequestUpdate error:用户未登陆");
return returnDataResponseEntity( "用户未登陆!");
}
int userId = user.getUserAccount().getId();
Map<String, Object> map = searchService.bulkRequestUpdate(cmsContents);
return returnDataResponseEntity(map);
}
}
service
package com.ge.es.hcapmelsservice.service;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.ge.es.hcapmelsservice.mapper.*;
import com.ge.es.hcapmelsservice.pojo.*;
import com.ge.es.hcapmelsservice.util.EsEnum;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
public class SearchService {
private static Logger log = LoggerFactory.getLogger(SearchService.class);
@Autowired
@Qualifier("restHighLevelClient")
private RestHighLevelClient client;
@Resource
CmsCollectionMapper cmsCollectionMapper;
@Resource
private CmsContentMapper cmsContentMapper;
@Resource
private CmsChannelMapper cmsChannelMapper;
@Resource
AccessCounterMapper accessCounterMapper;
@Resource
UserActivityLogMapper userActivityLogMapper;
private static List<CmsChannel> cmsChannels;
/**
* 查询接口
*
* @param content 查询内容
* @param channelId 频道号
* @param articleType 栏目类型
* @param userId token中的用户id
* @return Map<String, Object>
* @throws IOException
*/
public Map<String, Object> getSearchData(String content, String channelId, String articleType, int userId, int pageNum, String channelId2, String articleTypeFlag, String loginName,String modality,List<Integer> ids) throws IOException {
log.info("start SearchService.getSearchData parms is content={}, channelId={} articleType={} userId={}===pageNum={}==channelId2={}==articleTypeFlag={}==modality={}====", content, channelId, articleType, userId, pageNum, channelId2, articleTypeFlag,modality);
init();
return getMapsBy(content, channelId, articleType, userId, pageNum, channelId2, articleTypeFlag, loginName,modality, ids);
}
/**
* 删除 name = cms_index 的索引
*
* @throws IOException
*/
public Map<String, Object> testDeleteIndex() throws IOException {
Map<String, Object> map = new HashMap<>();
DeleteIndexRequest request = new DeleteIndexRequest(EsEnum.ES_CMS_INDEX.getValue());
AcknowledgedResponse delete = client.indices().delete(request, RequestOptions.DEFAULT);
log.info("start SearchService.testDeleteIndex isAcknowledged={}", delete.isAcknowledged());
map.put("1", delete.isAcknowledged());
return map;
}
public Map<String, Object> bulkRequestUpdate(List<CmsContent> cmsContents) throws IOException {
log.info("start SearchService.bulkRequestUpdate cmsContents.size={}", cmsContents.size());
Map<String, Object> returnMap = new HashMap();
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout("60s");
for (int i = 0; i < cmsContents.size(); i++) {
log.info("start SearchService.bulkRequestUpdate for.size={} cmsContent={}", i, cmsContents.get(i).toString());
//批量更新和批量删除，就在这里修改对应的请求就可以了
/*bulkRequest.add(
new UpdateRequest(EsEnum.ES_CMS_INDEX.getValue(),cmsContents.get(i).getId())
.doc(JSON.toJSONString(cmsContents.get(i)), XContentType.JSON)
);*/
bulkRequest.add(
new IndexRequest(EsEnum.ES_CMS_INDEX.getValue())
.id("" + cmsContents.get(i).getId())
.source(JSON.toJSONString(cmsContents.get(i)), XContentType.JSON)
);
}
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
log.info("end SearchService.bulkRequestUpdate return={} ", !bulkResponse.hasFailures());
returnMap.put("1", !bulkResponse.hasFailures());
return returnMap;
}
public void init() throws IOException {
GetIndexRequest request = new GetIndexRequest(EsEnum.ES_CMS_INDEX.getValue());
boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
log.info("SearchService.getSearchData exists={}", exists);
if (!exists) {
createIndex();
}
if (cmsChannels == null) {
List<CmsChannel> cmsChannel = cmsChannelMapper.selectList(null);
log.info("SearchService.getSearchData cmsChannels.size={}", cmsChannel.size());
if (cmsChannel.size() > 0) {
cmsChannels = cmsChannel;
}
}
}
public void createIndex() throws IOException {
//1.创建索引请求,像表格一样，作为数据的存储载体而存在
log.info("SearchService.getSearchData createIndex name={}", EsEnum.ES_CMS_INDEX.getValue());
/*CreateIndexRequest creatRequest = new CreateIndexRequest(EsEnum.ES_CMS_INDEX.getValue());
//2.执行请求 IndeicesClient,请求后获得相应
CreateIndexResponse createIndexResponse = client.indices().create(creatRequest, RequestOptions.DEFAULT);
log.info("SearchService.getSearchData createIndex createIndexResponse={} ",createIndexResponse);*/
//创建索引
boolean createIndexBool = createIndexByName();
if (!createIndexBool) {
log.info("SearchService.getSearchData createIndex false ==== ");
}
double num = cmsContentMapper.selectCount(new QueryWrapper<CmsContent>());//获取总条数
/* //查询同步的阅读量与收藏量
List<AccessCounterVo> viewCounList = accessCounterMapper.getViewCount();
Map<String, Long> viewCounMap = viewCounList.stream().collect(Collectors.toMap(AccessCounterVo::getId, AccessCounterVo::getCount));
List<AccessCounterVo> collectionCountList = accessCounterMapper.getCollectionCount();
Map<String, Long> collectionCountMap = collectionCountList.stream().collect(Collectors.toMap(AccessCounterVo::getId, AccessCounterVo::getCount));*/
log.info("SearchService.getSearchData createIndex num={} ", num);
double size = 10000;//每次查询数
double count = 0;
if (num <= 10000) {
IPage<CmsContent> cmsContentPage = new Page<>(1, (int) num);
cmsContentPage.setSize(-1);
cmsContentPage = cmsContentMapper.selectPage(cmsContentPage, null);
//bulkRequestData(cmsContentPage, viewCounMap, collectionCountMap);//初始化时需要同步阅读量与统计量
bulkRequestData(cmsContentPage, null, null);//初始化时不需要统计阅读量与统计量. 后两个参数传null
} else {
count = Math.ceil(num / size);
log.info("SearchService.getSearchData createIndex count={} ", count);
for (int i = 1; i <= count; i++) {
IPage<CmsContent> cmsContentPage = new Page<>(i, (int) size);
cmsContentPage = cmsContentMapper.selectPage(cmsContentPage, null);
//bulkRequestData(cmsContentPage, viewCounMap, collectionCountMap);//初始化时需要同步阅读量与统计量
bulkRequestData(cmsContentPage, null, null);//初始化时不需要统计阅读量与统计量.后两个参数传null
}
}
}
public void bulkRequestData(IPage<CmsContent> cmsContentPage, Map<String, Long> viewCounMap, Map<String, Long> collectionCountMap) throws IOException {
List<CmsContent> cmsContents = cmsContentPage.getRecords();
log.info("SearchService.getSearchData createIndex cmsContents.size(获取数据大小)={}", cmsContents.size());
putSortField(cmsContents, viewCounMap, collectionCountMap);
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout("60s");
for (int j = 0; j < cmsContents.size(); j++) {
//批量更新和批量删除，就在这里修改对应的请求就可以了
bulkRequest.add(new IndexRequest(EsEnum.ES_CMS_INDEX.getValue())
.id(cmsContents.get(j).getId())
.source(JSON.toJSONString(cmsContents.get(j)), XContentType.JSON));
}
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
log.info("SearchService.getSearchData createIndex bulkResponse.hasFailures(批量插入结果是否失败)={}", bulkResponse.hasFailures());
}
public void bulkRequestUpdate(IPage<CmsContent> cmsContentPage, Map<String, Long> viewCounMap, Map<String, Long> collectionCountMap) throws IOException {
List<CmsContent> cmsContents = cmsContentPage.getRecords();
log.info("SearchService.getSearchData createIndex cmsContents.size(获取数据大小)={}", cmsContents.size());
putSortField(cmsContents, viewCounMap, collectionCountMap);
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout("60s");
for (int j = 0; j < cmsContents.size(); j++) {
/*bulkRequest.add(new IndexRequest(EsEnum.ES_CMS_INDEX.getValue())
.id(cmsContents.get(j).getId())
.source(JSON.toJSONString(cmsContents.get(j)), XContentType.JSON));*/
//批量更新和批量删除，就在这里修改对应的请求就可以了
bulkRequest.add(
new UpdateRequest(EsEnum.ES_CMS_INDEX.getValue(), cmsContents.get(j).getId())
.doc(JSON.toJSONString(cmsContents.get(j)), XContentType.JSON)
);
}
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
log.info("SearchService.getSearchData createIndex bulkResponse.hasFailures(批量插入结果是否失败)={}", bulkResponse.hasFailures());
}
//获取文章的阅读量与搜藏量后放入es的实体中
public void putSortField(List<CmsContent> cmsContents, Map<String, Long> viewCounMap, Map<String, Long> collectionCountMap) {
for (CmsContent cmsContent : cmsContents) {
if (null != viewCounMap && null != viewCounMap.get(cmsContent.getId())) {
cmsContent.setViewCount(viewCounMap.get(cmsContent.getId()));
} else {
cmsContent.setViewCount(0L);
}
if (null !=collectionCountMap && null != collectionCountMap.get(cmsContent.getId())) {
cmsContent.setCollectionCount(collectionCountMap.get(cmsContent.getId()));
} else {
cmsContent.setCollectionCount(0L);
}
//入es库的时候转换tags为逗号隔开的name
List<Map> list = JSON.parseArray(cmsContent.getTags(), Map.class);
StringBuffer sb = new StringBuffer("");
if (null != list && list.size() > 0) {
for (int i = 0; i < list.size(); i++) {
Map map = list.get(i);
/*
if(i==(list.size()-1)){//最后一个
sb.append(map.get("name"));
}else{
sb.append(map.get("name")+",");
}*/
Iterator<String> iter = map.keySet().iterator();
while (iter.hasNext()) {
if (iter.next().equals("id")) {
iter.remove();
}
}
list.set(i, map);
}
String listJson = JSON.toJSONString(list);
cmsContent.setTags(listJson);
}
}
}
@Transactional
public Map<String, Object> getMapsBy(String content, String channelId, String articleType, int userId, int pageNmu, String channelId2, String articleTypeFlag, String loginName,String modality,List<Integer> ids) throws IOException {
log.info("SearchService.getSearchData.getMapsBy start content={}",content);
//content = content.replaceAll(" ", "");
int returnNum = insertHotWord(content, loginName);
Map<String, Object> map = new HashMap<String, Object>();
SearchRequest searchRequest = new SearchRequest(EsEnum.ES_CMS_INDEX.getValue());
//构建搜索条件
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//分页
sourceBuilder.from(0);
sourceBuilder.size(2);
//查询条件，可以使用QueryBuilders 工具来实现// QueryBuilders.termQuery 精确匹配//TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("tags", name); //单个匹配，搜索name为jack的文档
// 2.创建BoolQueryBuilder对象
//BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
// 3.设置boolQueryBuilder条件 matchPhraseQuery:短句查询 matchQuery:精确查询
MatchQueryBuilder matchPhraseQueryBuilder = QueryBuilders.matchQuery(EsEnum.ES_CMS_TAGS.getValue(), content);
MatchQueryBuilder matchPhraseQueryBuilder2 = QueryBuilders.matchQuery(EsEnum.ES_CMS_TITLE.getValue(), content);
MatchQueryBuilder matchPhraseQueryBuilder3 = QueryBuilders.matchQuery(EsEnum.ES_CMS_CONTENT.getValue(), content);
MatchQueryBuilder matchPhraseQueryBuilder4 = QueryBuilders.matchQuery(EsEnum.ES_CMS_SUMMARY.getValue(), content);
MatchPhraseQueryBuilder matchPhraseQueryBuilder5 = QueryBuilders.matchPhraseQuery(EsEnum.ES_CMS_CONTENT.getValue(), content);
MatchPhraseQueryBuilder matchPhraseQueryBuilder6 = QueryBuilders.matchPhraseQuery(EsEnum.ES_CMS_SUMMARY.getValue(), content);
MatchPhraseQueryBuilder matchPhraseQueryBuilder7 = QueryBuilders.matchPhraseQuery(EsEnum.ES_CMS_TITLE.getValue(), content);
MatchPhraseQueryBuilder matchPhraseQueryBuilder8 = QueryBuilders.matchPhraseQuery(EsEnum.ES_CMS_TAGS.getValue(), content);
// 子boolQueryBuilder条件条件，用来表示查询条件or的关系
BoolQueryBuilder childBoolQueryBuilder = new BoolQueryBuilder()
.should(matchPhraseQueryBuilder)
.should(matchPhraseQueryBuilder2)
.should(matchPhraseQueryBuilder3)
.should(matchPhraseQueryBuilder4)
.should(matchPhraseQueryBuilder5)
.should(matchPhraseQueryBuilder6)
.should(matchPhraseQueryBuilder7)
.should(matchPhraseQueryBuilder8);
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(EsEnum.ES_CMS_OFFSHELF.getValue(), 1);
TermQueryBuilder termQueryBuilder2 = QueryBuilders.termQuery(EsEnum.ES_CMS_SOURCETYPE.getValue(), 2);
//封装批量查询ids的BoolQueryBuilder
BoolQueryBuilder idsBool = getIdsBoolQueryBuilder(ids);
//TermQueryBuilder termQueryBuilder6 = null;
//MatchQueryBuilder termQueryBuilder6 = null;
MatchPhraseQueryBuilder termQueryBuilder6=null;
if (null != modality && !"all".equals(modality)) {
//modality = modality.toLowerCase();
//termQueryBuilder6 = QueryBuilders.termQuery("modality", modality);
//termQueryBuilder6 = QueryBuilders.matchQuery("modality", modality);
termQueryBuilder6 = QueryBuilders.matchPhraseQuery("modality", modality);
}
//channel判断是否查询全部
if ("0".equals(channelId)) {
// 4.添加查询条件到boolQueryBuilder中 must:必须出现 filter:参与但不算分 must_not:排除在外 should：应该出现，不一定同时出现 .minimumShouldMatch(1):小于1分的排除在外
if ("0".equals(articleType)) {//类别与类型都没有选
if(null != ids){//返回以channl为聚合的集合
sourceBuilder.size(3);
if("".equals(content)){
sourceBuilder.sort(new FieldSortBuilder("pubTime").order(SortOrder.DESC));
}
log.info("SearchService.getSearchData.getMapsBy channelId is '' and articleType is 0 ");
//TermQueryBuilder termQueryBuilder3 = QueryBuilders.termQuery("articleType", 1);
Map totalNumMap = new HashMap();
List<Map<String, Object>> list1 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, null, QueryBuilders.termQuery("channel", 1), userId, totalNumMap, null, content,termQueryBuilder6, idsBool);
Map totalNumMap2 = new HashMap();
List<Map<String, Object>> list2 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, null, QueryBuilders.termQuery("channel", 2), userId, totalNumMap2, null, content,termQueryBuilder6, idsBool);
Map totalNumMap3 = new HashMap();
List<Map<String, Object>> list3 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, null, QueryBuilders.termQuery("channel", 3), userId, totalNumMap3, null, content,termQueryBuilder6, idsBool);
Map totalNumMap4 = new HashMap();
List<Map<String, Object>> list4 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, null, QueryBuilders.termQuery("channel", 4), userId, totalNumMap4, null, content,termQueryBuilder6, idsBool);
buildReturnMap(map, totalNumMap, list1, "1", "1");
buildReturnMap(map, totalNumMap2, list2, "2", "2");
buildReturnMap(map, totalNumMap3, list3, "3", "3");
buildReturnMap(map, totalNumMap4, list4, "4", "4");
}else{
if("".equals(content)){
sourceBuilder.sort(new FieldSortBuilder("viewCount").order(SortOrder.DESC));
sourceBuilder.sort(new FieldSortBuilder("collectionCount").order(SortOrder.DESC));
sourceBuilder.sort(new FieldSortBuilder("pubTime").order(SortOrder.DESC));
}
log.info("SearchService.getSearchData.getMapsBy channelId is '' and articleType is 0 ");
//TermQueryBuilder termQueryBuilder3 = QueryBuilders.termQuery("articleType", 1);
Map totalNumMap = new HashMap();
List<Map<String, Object>> list1 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, QueryBuilders.termQuery("articleType", 1), null, userId, totalNumMap, null, content,termQueryBuilder6, idsBool);
Map totalNumMap2 = new HashMap();
List<Map<String, Object>> list2 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, QueryBuilders.termQuery("articleType", 3), null, userId, totalNumMap2, null, content,termQueryBuilder6, idsBool);
Map totalNumMap3 = new HashMap();
List<Map<String, Object>> list3 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, QueryBuilders.termQuery("articleType", 2), null, userId, totalNumMap3, null, content,termQueryBuilder6, idsBool);
Map totalNumMap4 = new HashMap();
List<Map<String, Object>> list4 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, QueryBuilders.termQuery("articleType", 4), null, userId, totalNumMap4, null, content,termQueryBuilder6, idsBool);
buildReturnMap(map, totalNumMap, list1, "1", "1");
buildReturnMap(map, totalNumMap2, list2, "3", "3");
buildReturnMap(map, totalNumMap3, list3, "2", "2");
buildReturnMap(map, totalNumMap4, list4, "4", "4");
}
} else {
if("".equals(content)){
sourceBuilder.sort(new FieldSortBuilder("pubTime").order(SortOrder.DESC));
sourceBuilder.sort(new FieldSortBuilder("viewCount").order(SortOrder.DESC));
}
log.info("SearchService.getSearchData.getMapsBy channelId is '' and articleType is not 0 ");
TermQueryBuilder termQueryBuilder3 = QueryBuilders.termQuery("articleType", Integer.parseInt(articleType));
sourceBuilder.from(getFromPageNum(pageNmu));
sourceBuilder.size(10);
Map totalNumMap = new HashMap();
List<Map<String, Object>> list1 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, termQueryBuilder3, null, userId, totalNumMap, null, content,termQueryBuilder6, idsBool);
buildReturnMap(map, totalNumMap, list1, articleType, articleType);
}
} else {
TermQueryBuilder termQueryBuilder4 = QueryBuilders.termQuery("channel", Integer.parseInt(channelId));
TermQueryBuilder termQueryBuilder5 = null;
if (!"0".equals(channelId2)) {
termQueryBuilder5 = QueryBuilders.termQuery("channel2", Integer.parseInt(channelId2));
}
if("".equals(content)){
sourceBuilder.sort(new FieldSortBuilder("pubTime").order(SortOrder.DESC));
sourceBuilder.sort(new FieldSortBuilder("viewCount").order(SortOrder.DESC));
}
if ("0".equals(articleType)) {//类别与类型都没有选
if ("1".equals(articleTypeFlag)) {//articleTypeFlag 1: 当articleType为1时 返回每组两条的所有数据 articleTypeFlag=0:当 当articleType为0时 返回不分组的数据
log.info("SearchService.getSearchData.getMapsBy articleTypeFlag=1 ");
Map totalNumMap = new HashMap();
List<Map<String, Object>> list1 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, QueryBuilders.termQuery("articleType", 1), termQueryBuilder4, userId, totalNumMap, termQueryBuilder5, content,termQueryBuilder6, idsBool);
Map totalNumMap2 = new HashMap();
List<Map<String, Object>> list2 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, QueryBuilders.termQuery("articleType", 3), termQueryBuilder4, userId, totalNumMap2, termQueryBuilder5, content,termQueryBuilder6, idsBool);
Map totalNumMap3 = new HashMap();
List<Map<String, Object>> list3 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, QueryBuilders.termQuery("articleType", 2), termQueryBuilder4, userId, totalNumMap3, termQueryBuilder5, content,termQueryBuilder6, idsBool);
Map totalNumMap4 = new HashMap();
List<Map<String, Object>> list4 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, QueryBuilders.termQuery("articleType", 4), termQueryBuilder4, userId, totalNumMap4, termQueryBuilder5, content,termQueryBuilder6, idsBool);
buildReturnMap(map, totalNumMap, list1, "1", "1");
buildReturnMap(map, totalNumMap2, list2, "3", "3");
buildReturnMap(map, totalNumMap3, list3, "2", "2");
buildReturnMap(map, totalNumMap4, list4, "4", "4");
} else {
log.info("SearchService.getSearchData.getMapsBy channelId is not '' and articleType is 0 ");
Map totalNumMap = new HashMap();
sourceBuilder.from(getFromPageNum(pageNmu));
sourceBuilder.size(10);
List<Map<String, Object>> list1 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, null, termQueryBuilder4, userId, totalNumMap, termQueryBuilder5, content,termQueryBuilder6, idsBool);
buildReturnMap(map, totalNumMap, list1, articleType, channelId);
}
} else {
log.info("SearchService.getSearchData.getMapsBy channelId is not '' and articleType is not 0 ");
TermQueryBuilder termQueryBuilder3 = QueryBuilders.termQuery("articleType", Integer.parseInt(articleType));
sourceBuilder.from(getFromPageNum(pageNmu));
sourceBuilder.size(10);
Map totalNumMap = new HashMap();
List<Map<String, Object>> list1 = getList(searchRequest, sourceBuilder, childBoolQueryBuilder, termQueryBuilder, termQueryBuilder2, termQueryBuilder3, termQueryBuilder4, userId, totalNumMap, termQueryBuilder5, content,termQueryBuilder6, idsBool);
buildReturnMap(map, totalNumMap, list1, articleType, channelId);
}
}
return map;
}
private BoolQueryBuilder getIdsBoolQueryBuilder(List<Integer> ids) {
BoolQueryBuilder idsBool =null;
if(null != ids){
log.info("SearchService.ids={}",ids.toString());
idsBool = new BoolQueryBuilder();
for (int i = 0; i < ids.size(); i++) {
idsBool.should(QueryBuilders.termQuery(EsEnum.ES_CMS_ID.getValue(), ids.get(i)));
}
}
return idsBool;
}
public int insertHotWord(String content, String loginName) {
log.info("SearchService.getSearchData.getMapsBy insertHotWord start content={} ,loginName={}", content,loginName);
if("".equals(content)|| null == content){
log.info("SearchService.getSearchData.getMapsBy insertHotWord start content is null");
return 0;
}
int num = 0;
try {
//1.搜索内容入库
num = searchContentToDb(content, loginName);
//2.查询索引，如果没有则创建
log.info("SearchService.getSearchData.getMapsBy insertHotWord 2.cmsHotWordIndex=====searchContentToDb return={}===",num);
cmsHotWordIndex();
//3.通过ik分词器把内容进行分词操作
List<UserActivityLogEsVo> userActivityLogEsVoList = ikToParticiple(content, loginName);
//4.得到分词结果后进行入es的操作
participleToEs(userActivityLogEsVoList);
} catch (Exception e) {
e.printStackTrace();
}
return num;
}
private int searchContentToDb(String content, String loginName) {
UserActivityLog userActivityLog = new UserActivityLog();
//userActivityLog.setContent("内容测试2");
userActivityLog.setLoginName(loginName);
userActivityLog.setTarget(content);
userActivityLog.setTime(new Date());
userActivityLog.setType(EsEnum.ES_CMS_HOTWORD.getValue());
return userActivityLogMapper.insert(userActivityLog);
}
private void participleToEs(List<UserActivityLogEsVo> userActivityLogEsVoList) throws IOException {
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout("60s");
for (int i = 0; i < userActivityLogEsVoList.size(); i++) {
log.info("start SearchService.bulkRequestUpdate for.size={} cmsContent={}", i, userActivityLogEsVoList.get(i).toString());
//批量更新和批量删除，就在这里修改对应的请求就可以了
bulkRequest.add(
new IndexRequest(EsEnum.ES_CMS_CMSHOTWORDINDEX.getValue())
//.id("" + UserActivityLogEsVoList.get(i).getId())
.source(JSON.toJSONString(userActivityLogEsVoList.get(i)), XContentType.JSON)
);
}
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
log.info("end SearchService.bulkRequestUpdate return={} ", !bulkResponse.hasFailures());
}
private List<UserActivityLogEsVo> ikToParticiple(String content, String loginName) throws IOException {
List<UserActivityLogEsVo> userActivityLogEsVoList = new ArrayList<UserActivityLogEsVo>();
Request request2 = new Request("GET", "_analyze");
JSONObject entity = new JSONObject();
//entity.put("analyzer", "ik_max_word");
entity.put("analyzer", "ik_smart");
entity.put("text", content);
request2.setJsonEntity(entity.toJSONString());
Response response = this.client.getLowLevelClient().performRequest(request2);
JSONObject tokens = JSONObject.parseObject(EntityUtils.toString(response.getEntity()));
JSONArray arrays = tokens.getJSONArray("tokens");
for (int i = 0; i < arrays.size(); i++) {
UserActivityLogEsVo userActivityLogEsVo = new UserActivityLogEsVo();
JSONObject obj = JSON.parseObject(arrays.getString(i));
if (obj.getString("token").length() > 1) {
System.out.println(arrays.getString(i));
userActivityLogEsVo.setTarget(obj.getString("token"));
userActivityLogEsVo.setType("hotWord");
userActivityLogEsVo.setLoginName(loginName);
userActivityLogEsVo.setContent("内容字段");
userActivityLogEsVo.setSearchTime(System.currentTimeMillis());
//userActivityLogEsVo.setQuantity(1L);
userActivityLogEsVoList.add(userActivityLogEsVo);
}
}
return userActivityLogEsVoList;
}
//创建索引 cms_hotWord_index
public boolean cmsHotWordIndex() {
boolean acknowledged = false;
try {
GetIndexRequest request = new GetIndexRequest(EsEnum.ES_CMS_CMSHOTWORDINDEX.getValue());
boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
log.info("SearchService.cmsHotWordIndex insertHotWord={}", exists);
if (!exists) {
log.info("SearchService.cmsHotWordIndex insertHotWord start");
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.field("properties")
.startObject()
.field("id").startObject().field("index", "true").field("type", "long").endObject()
.field("quantity").startObject().field("index", "true").field("type", "long").endObject()
.field("searchTime").startObject().field("index", "true").field("type", "long").endObject()
.field("type").startObject().field("index", "true").field("type", "text").endObject()
.field("loginName").startObject().field("index", "true").field("type", "text").endObject()
.endObject()
.endObject();
CreateIndexRequest createIndexRequest = new CreateIndexRequest(EsEnum.ES_CMS_CMSHOTWORDINDEX.getValue());
createIndexRequest.mapping(builder);
CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
acknowledged = createIndexResponse.isAcknowledged();
log.info("SearchService.cmsHotWordIndex insertHotWord end");
}
} catch (Exception e) {
e.printStackTrace();
}
return acknowledged;
}
public void buildReturnMap(Map<String, Object> map, Map totalNumMap, List<Map<String, Object>> list1, String s, String s2) {
SearchDataVo searchDataVo = new SearchDataVo();
searchDataVo.setSearchData(list1);
searchDataVo.setTotalNum((String) totalNumMap.get("totalNum"));
searchDataVo.setArticleType(s);
map.put(s2, searchDataVo);
}
public List<Map<String, Object>> getList(SearchRequest searchRequest, SearchSourceBuilder sourceBuilder, BoolQueryBuilder childBoolQueryBuilder, TermQueryBuilder termQueryBuilder, TermQueryBuilder termQueryBuilder2, TermQueryBuilder termQueryBuilder3, TermQueryBuilder termQueryBuilder4, int userId, Map totalNumMap, TermQueryBuilder termQueryBuilder5, String content, MatchPhraseQueryBuilder termQueryBuilder6,BoolQueryBuilder idsBool) throws IOException {
List<Map<String, Object>> list = new ArrayList();
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
assemblyBoolQueryBuilder(childBoolQueryBuilder, termQueryBuilder3, termQueryBuilder4, termQueryBuilder5, boolQueryBuilder, content, termQueryBuilder, termQueryBuilder6, idsBool);
//高亮初始化
HighlightBuilder highlightBuilder = highlightBuilderInit();
highlightBuilder.fragmentSize(800000);
highlightBuilder.numOfFragments(0);
searchSourceBuilderinit(sourceBuilder, boolQueryBuilder, highlightBuilder);
searchRequest.source(sourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
totalNumMap.put("totalNum", searchResponse.getHits().getTotalHits().value + "");
for (SearchHit documentFields : searchResponse.getHits().getHits()) {
//System.out.println(documentFields.getSourceAsMap());
//System.out.println(documentFields.getSourceRef());
Map<String, Object> sourceAsMap = documentFields.getSourceAsMap();
isCollection(userId, sourceAsMap);//判断该文章是否被用户收藏
//高亮
highlightField(list, documentFields, sourceAsMap);
}
return list;
}
public void assemblyBoolQueryBuilder(BoolQueryBuilder childBoolQueryBuilder, TermQueryBuilder termQueryBuilder3, TermQueryBuilder termQueryBuilder4, TermQueryBuilder termQueryBuilder5, BoolQueryBuilder boolQueryBuilder, String content, TermQueryBuilder termQueryBuilder,MatchPhraseQueryBuilder termQueryBuilder6,BoolQueryBuilder idsBool) {
TermQueryBuilder termQueryBuilder2 = QueryBuilders.termQuery(EsEnum.ES_CMS_STATUS.getValue(), 1);//状态必须是发布的
if ("".equals(content)) {
boolQueryBuilder.should(childBoolQueryBuilder).must(termQueryBuilder).must(termQueryBuilder2);
packagingEsConditions(termQueryBuilder3, termQueryBuilder4, termQueryBuilder5, boolQueryBuilder, termQueryBuilder6, idsBool);
}else{
boolQueryBuilder.must(childBoolQueryBuilder).must(termQueryBuilder).must(termQueryBuilder2);
packagingEsConditions(termQueryBuilder3, termQueryBuilder4, termQueryBuilder5, boolQueryBuilder, termQueryBuilder6, idsBool);
}
}
//封装es条件
private void packagingEsConditions(TermQueryBuilder termQueryBuilder3, TermQueryBuilder termQueryBuilder4, TermQueryBuilder termQueryBuilder5, BoolQueryBuilder boolQueryBuilder, MatchPhraseQueryBuilder termQueryBuilder6,BoolQueryBuilder idsBool) {
if (null != termQueryBuilder3) {//判断是否有articleType
boolQueryBuilder.must(termQueryBuilder3);
}
if (null != termQueryBuilder4) {//判断是否有channel
boolQueryBuilder.must(termQueryBuilder4);
if (null != termQueryBuilder5) {//判断是否有channel2
boolQueryBuilder.must(termQueryBuilder5);
}
}
if (null != termQueryBuilder6) {//判断是否有modality
boolQueryBuilder.must(termQueryBuilder6);
}
if (null != idsBool) {//判断是否有modality
boolQueryBuilder.must(idsBool);
}
}
public void searchSourceBuilderinit(SearchSourceBuilder sourceBuilder, BoolQueryBuilder boolQueryBuilder, HighlightBuilder highlightBuilder) {
sourceBuilder.highlighter(highlightBuilder);
sourceBuilder.query(boolQueryBuilder);
sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
}
public HighlightBuilder highlightBuilderInit() {
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.requireFieldMatch(true);
highlightBuilder.field(EsEnum.ES_CMS_TITLE.getValue());
highlightBuilder.field(EsEnum.ES_CMS_CONTENT.getValue());
highlightBuilder.field(EsEnum.ES_CMS_TAGS.getValue());
highlightBuilder.field(EsEnum.ES_CMS_SUMMARY.getValue());
highlightBuilder.preTags(EsEnum.ES_CMS_HIGHLIGHTPREFIX.getValue());
highlightBuilder.postTags(EsEnum.ES_CMS_HIGHLIGHTSUFFIX.getValue());
/*highlightBuilder.preTags("<span style='color:red'>");
highlightBuilder.postTags("</span>");*/
return highlightBuilder;
}
//高亮字段增加特殊标签
public void highlightField(List<Map<String, Object>> list, SearchHit documentFields, Map<String, Object> sourceAsMap) {
Map<String, HighlightField> highlightFields = documentFields.getHighlightFields();
HighlightField title = highlightFields.get(EsEnum.ES_CMS_TITLE.getValue());
HighlightField contentStr = highlightFields.get(EsEnum.ES_CMS_CONTENT.getValue());
HighlightField tag = highlightFields.get(EsEnum.ES_CMS_TAGS.getValue());
HighlightField summary = highlightFields.get(EsEnum.ES_CMS_SUMMARY.getValue());
//解析高亮的字段，将原来的字段替换为我们高亮的字段即可
if (title != null) {
Text[] fragments = title.fragments();
String n_title = "";
for (Text text : fragments) {
n_title += text;
}
sourceAsMap.put(EsEnum.ES_CMS_TITLE.getValue(), n_title);
}
if (contentStr != null) {
Text[] fragments = contentStr.fragments();
String n_contentStr = "";
for (Text text : fragments) {
n_contentStr += text;
}
sourceAsMap.put(EsEnum.ES_CMS_CONTENT.getValue(), n_contentStr);
}
if (tag != null) {
Text[] fragments = tag.fragments();
String n_tags = "";
for (Text text : fragments) {
n_tags += text;
}
sourceAsMap.put(EsEnum.ES_CMS_TAGS.getValue(), n_tags);
}
if (summary != null) {
Text[] fragments = summary.fragments();
String n_summary = "";
for (Text text : fragments) {
n_summary += text;
}
sourceAsMap.put(EsEnum.ES_CMS_SUMMARY.getValue(), n_summary);
}
list.add(sourceAsMap);
}
//判断用户是否收藏过该文章，收藏就增加字段属性为true 否则为false
public void isCollection(int userId, Map<String, Object> sourceAsMap) {
//判断用户是否收藏过该文章
List<CmsCollection> cmsContent = cmsCollectionMapper.getContentIdByUId(userId);
Map cmsContentMap = cmsContent.stream().collect(Collectors.toMap(CmsCollection::getContentId, CmsCollection::getUserId));
Long id = Long.parseLong((String) sourceAsMap.get("id"));
if (cmsContentMap.containsKey(id)) {
sourceAsMap.put("collection", true);
} else {
sourceAsMap.put("collection", false);
}
if (cmsChannels != null) {
Map cmsChannelsMap = cmsChannels.stream().collect(Collectors.toMap(CmsChannel::getId, CmsChannel::getName));
Long channelId = Long.valueOf(String.valueOf(sourceAsMap.get("channel")));
if (cmsChannelsMap.containsKey(channelId)) {
sourceAsMap.put("channelName", cmsChannelsMap.get(channelId));
}
}
}
public int getFromPageNum(int pageNum) {
pageNum++;
int pageSize = 10;
int pageNum2 = (pageNum - 1) * pageSize;//==0?1:(pageNum-1)*pageSize
return pageNum2;
}
public boolean createIndexByName() throws IOException {
//type:keyword 不分词
log.info("==开始初始化索引=");
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.field("properties")
.startObject()
.field("id").startObject().field("index", "true").field("type", "long").endObject()
.field("pubTime").startObject().field("index", "true").field("type", "long").endObject()
.field("channel").startObject().field("index", "true").field("type", "long").endObject()
.field("channel2").startObject().field("index", "true").field("type", "long").endObject()
.field("inertTime").startObject().field("index", "true").field("type", "long").endObject()
.field("articleType").startObject().field("index", "true").field("type", "long").endObject()
.field("modality").startObject().field("index", "true").field("type", "text").field("analyzer", "ik_smart").field("search_analyzer", "ik_smart").endObject()
//.field("modality").startObject().field("index", "true").field("type", "text").endObject()
.field("sourceType").startObject().field("index", "true").field("type", "long").endObject()
.field("collectionCount").startObject().field("index", "true").field("type", "long").endObject()
.field("viewCount").startObject().field("index", "true").field("type", "long").endObject()
.field("status").startObject().field("index", "true").field("type", "long").endObject()
.field("offShelf").startObject().field("index", "true").field("type", "long").endObject()
.field("fingerprintId").startObject().field("index", "true").field("type", "text").endObject()
.field("articleUrl").startObject().field("index", "true").field("type", "text").endObject()
.field("creator").startObject().field("index", "true").field("type", "text").endObject()
.field("cover").startObject().field("index", "true").field("type", "text").endObject()
.field("title").startObject().field("index", "true").field("type", "text").field("analyzer", "ik_smart").field("search_analyzer", "ik_smart").endObject()
.field("content").startObject().field("index", "true").field("type", "text").field("analyzer", "ik_smart").field("search_analyzer", "ik_smart").endObject()
.field("summary").startObject().field("index", "true").field("type", "text").field("analyzer", "ik_smart").field("search_analyzer", "ik_smart").endObject()
.field("tags").startObject().field("index", "true").field("type", "text").field("analyzer", "ik_smart").field("search_analyzer", "ik_smart").endObject()
.endObject()
.endObject();
CreateIndexRequest createIndexRequest = new CreateIndexRequest(EsEnum.ES_CMS_INDEX.getValue());
createIndexRequest.mapping(builder);
CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
boolean acknowledged = createIndexResponse.isAcknowledged();
return acknowledged;
}
}
聚合操作相关
package com.ge.es.hcapmelsservice.controller;
import com.ge.es.hcapmelsservice.pojo.EEMonitorData;
import com.ge.es.hcapmelsservice.service.EeMonitorDataService;
import com.ge.es.hcapmelsservice.service.UserProfileService;
import com.ge.es.hcapmelsservice.util.BizStatusCode;
import com.ge.es.hcapmelsservice.util.ResponseData;
import com.google.common.collect.ImmutableMap;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.format.annotation.DateTimeFormat;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.util.Date;
import java.util.List;
import java.util.Map;
@Api("EeMonitorDataController")
@RestController
@RequestMapping("/api/apm/monitor/search/")
public class EeMonitorDataController {
private static Logger log = LoggerFactory.getLogger(EeMonitorDataController.class);
@Autowired
private UserProfileService userProfileService;
@Autowired
private EeMonitorDataService eeMonitorDataService;
@ResponseBody
@RequestMapping(value = "/bulkUpdate",method = RequestMethod.POST)
@ApiOperation(value = "批量更新")
public ResponseEntity<ResponseData> BulkRequestUpdate(
@RequestHeader(value = "Authorization") String authorization,
@RequestBody List<EEMonitorData> eeMonitorData
) throws IOException {
Map<String, Object> map = eeMonitorDataService.bulkRequestUpdate(eeMonitorData);
return ResponseEntity.ok().body(
new ResponseData(
BizStatusCode.OK,
"The EeMonitorDataController was created successfully!",
null,
"update success"
)
);
}
@RequestMapping(value = "/getSearch", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
@ApiOperation(value = "通过搜索内容获取文档信息")
public ResponseEntity<ResponseData> search3(
@RequestHeader(value = "Authorization") String token,
@RequestParam(name = "assetUid") String assetUid,
@RequestParam(name = "startTime", required = false) @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") Date startTime,
@RequestParam(name = "endTime", required = false) @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss") Date endTime,
@RequestParam(name = "interval", required = false) Integer interval
){
ImmutableMap<String, Object> map = eeMonitorDataService.getSearchData(assetUid,startTime,endTime,interval);
return ResponseEntity.ok().body(
new ResponseData(
BizStatusCode.OK,
"The EeMonitorDataController was created successfully!",
null,
map
)
);
}
@RequestMapping(value = "/getSearchByAssetUids", method = RequestMethod.GET, produces = MediaType.APPLICATION_JSON_VALUE)
@ApiOperation(value = "通过AssetUIds获取最近的文档信息")
public ResponseEntity<ResponseData> getSearchByAssetUids(
@RequestParam(value = "assetUids") List<String> assetUids
) throws IOException {
List<Map<String, Object>> eEMonitorDataMap = eeMonitorDataService.getSearchByAssetUids(assetUids);
return ResponseEntity.ok().body(
new ResponseData(
BizStatusCode.OK,
"The EeMonitorDataController.getSearchByAssetUids was created successfully!",
null,
eEMonitorDataMap
)
);
}
}
package com.ge.es.hcapmelsservice.service;
import com.alibaba.fastjson.JSON;
import com.ge.es.hcapmelsservice.pojo.*;
import com.ge.es.hcapmelsservice.util.CaptureDataUtils;
import com.google.common.collect.ImmutableMap;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.CreateIndexResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.aggregations.*;
import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
import org.elasticsearch.search.aggregations.metrics.*;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.text.DecimalFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
public class EeMonitorDataService {
private static Logger log = LoggerFactory.getLogger(EeMonitorDataService.class);
private static final DecimalFormat df = new DecimalFormat( "0.##" );
@Autowired
@Qualifier("restHighLevelClient")
private RestHighLevelClient client;
public Map<String, Object> bulkRequestUpdate(List<EEMonitorData> eEMonitorData) throws IOException {
//判断需要初始化库不需要
init();
Map<String, Object> map = updateEEMonitorData(eEMonitorData);
return map;
}
public void init() throws IOException {
GetIndexRequest request = new GetIndexRequest("ee_monitor_data");
boolean exists = client.indices().exists(request, RequestOptions.DEFAULT);
log.info("EeMonitorDataService.init exists={}", exists);
if (!exists) {
createIndex();
}
}
public void createIndex() throws IOException {
//1.创建索引请求,像表格一样，作为数据的存储载体而存在
boolean createIndexBool = createMonitor();
if (!createIndexBool) {
log.info("EeMonitorDataService.createIndex false ");
}
}
public boolean createMonitor() throws IOException {
log.info("EeMonitorDataService.createMonitor name={}", "ee_monitor_data");
XContentBuilder builder = XContentFactory.jsonBuilder()
.startObject()
.field("properties")
.startObject()
.field("id").startObject().field("index", "true").field("type", "text").endObject()
.field("time").startObject().field("index", "true").field("type", "date").endObject()
.field("systemID").startObject().field("index", "true").field("type", "text").endObject()
.field("assetUid").startObject().field("index", "true").field("type", "text").endObject()
.field("subDistrict").startObject().field("index", "true").field("type", "text").endObject()
.field("hospital").startObject().field("index", "true").field("type", "text").endObject()
.field("temperature").startObject().field("index", "true").field("type", "double").endObject()
.field("humidity").startObject().field("index", "true").field("type", "double").endObject()
.field("groundingCurrent").startObject().field("index", "true").field("type", "double").endObject()
.field("compressorStatus").startObject().field("index", "true").field("type", "long").endObject()
.field("result").startObject().field("index", "true").field("type", "text").endObject()
.field("resultTemperature").startObject().field("index", "true").field("type", "long").endObject()
.field("resultHumidity").startObject().field("index", "true").field("type", "long").endObject()
.field("resultCompressorStatus").startObject().field("index", "true").field("type", "long").endObject()
.field("resultGroundingCurrent").startObject().field("index", "true").field("type", "long").endObject()
.endObject()
.endObject();
CreateIndexRequest createIndexRequest = new CreateIndexRequest("ee_monitor_data");
// createIndexRequest.settings(Settings.builder().put("number_of_shards", "1").put("number_of_replicas", "0"));
createIndexRequest.mapping(builder);
CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
boolean acknowledged = createIndexResponse.isAcknowledged();
return acknowledged;
}
public Map<String, Object> updateEEMonitorData(List<EEMonitorData> eEMonitorData) throws IOException {
log.info("start EeMonitorDataService.updateEEMonitorData cmsContents.size={}", eEMonitorData.size());
Map<String, Object> returnMap = new HashMap();
BulkRequest bulkRequest = new BulkRequest();
bulkRequest.timeout("60s");
for (int i = 0; i < eEMonitorData.size(); i++) {
bulkRequest.add(
new IndexRequest("ee_monitor_data")
.id(eEMonitorData.get(i).getId())
.source(JSON.toJSONString(eEMonitorData.get(i)), XContentType.JSON)
);
}
BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
log.info("end EeMonitorDataService.updateEEMonitorData return={} ", !bulkResponse.hasFailures());
returnMap.put("1", !bulkResponse.hasFailures());
return returnMap;
}
public ImmutableMap<String, Object> getSearchData(String assetUid, Date startTime, Date endTime, Integer interval) {
try {
//1.构建聚合条件
SearchResponse searchResponse = getSearchResponse(assetUid, interval, startTime, endTime);
//1.获取聚合的结果
List<EEMonitorData> eeMonitorDataList = getEeMonitorData(searchResponse);
//3.处理聚合的结果
ImmutableMap<String, Object> result = getStringObjectImmutableMap(interval, startTime, endTime, eeMonitorDataList);
return result;
} catch (IOException e) {
e.printStackTrace();
} catch (ParseException e) {
e.printStackTrace();
}
return null;
}
private ImmutableMap<String, Object> getStringObjectImmutableMap(Integer interval, Date startDate, Date endDate, List<EEMonitorData> eeMonitorDataList) {
Iterator<EEMonitorData> it = eeMonitorDataList.iterator();
while(it.hasNext()){
EEMonitorData x = it.next();
if("0.0".equals(x.getTemperature()+"") &&"0.0".equals(x.getHumidity()+"") &&"0.0".equals(x.getGroundingCurrent()+"")){
it.remove();
}
}
eeMonitorDataList.sort((u1,u2) -> u1.getTime().compareTo(u2.getTime()));
List<ImmutableMap<String, Object>> items;
items = eeMonitorDataList.stream().map(eeMonitorData -> new ImmutableMap.Builder<String, Object>()
.put("time", eeMonitorData.getTime() == null ? "" : new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(eeMonitorData.getTime()))
.put("temperature", formatAsTwoDigits(eeMonitorData.getTemperature()))
.put("humidity", formatAsTwoDigits(eeMonitorData.getHumidity()))
.put("groundingCurrent", formatAsTwoDigits(eeMonitorData.getGroundingCurrent()))
.put("status", eeMonitorData.getCompressorStatus())
.put("resultCompressorStatus", translate(eeMonitorData.getResultCompressorStatus()))
.put("resultGroundingCurrent", translate(eeMonitorData.getResultGroundingCurrent()))
.put("resultTemperature", translate(eeMonitorData.getResultTemperature()))
.put("resultHumidity", translate(eeMonitorData.getResultHumidity()))
.build()).collect(Collectors.toList());
ImmutableMap<String, Object> data2 = new ImmutableMap.Builder<String, Object>()
.put("startTime", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(startDate))
.put("endTime", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(endDate))
.put("interval", interval)
.put("items", items)
.build();
return new ImmutableMap.Builder<String, Object>().put("data", data2).build();
}
private List<EEMonitorData> getEeMonitorData(SearchResponse searchResponse) throws ParseException {
List<ImmutableMap<String, Object>> items;
List<EEMonitorData> eeMonitorDataList = new ArrayList<>();
Aggregations aggregations = searchResponse.getAggregations();
Aggregation aggregation1 = aggregations.get("by_time");
List<? extends Histogram.Bucket> buckets = ((Histogram) aggregation1).getBuckets();
for (Histogram.Bucket bucket : buckets) {
EEMonitorData data = new EEMonitorData();
String keyAsString = bucket.getKeyAsString();
Sum sumby_humidity = bucket.getAggregations().get("by_humidity");
ParsedAvg avgby_humidity2 = bucket.getAggregations().get("by_humidity2");
ParsedAvg groundingCurrent = bucket.getAggregations().get("by_groundingCurrent");
ParsedAvg temperature = bucket.getAggregations().get("by_temperature");
ParsedMax compressorStatus = bucket.getAggregations().get("by_compressorStatus");
ParsedMax resultCompressorStatus = bucket.getAggregations().get("by_resultCompressorStatus");
ParsedMax resultTemperature = bucket.getAggregations().get("by_resultTemperature");
ParsedMax resultHumidity = bucket.getAggregations().get("by_resultHumidity");
ParsedMax resultGroundingCurrent = bucket.getAggregations().get("by_resultGroundingCurrent");
data.setGroundingCurrent(typeConversionToDouble(groundingCurrent.getValue()+""));
data.setHumidity(typeConversionToDouble(avgby_humidity2.getValue()+""));
data.setTemperature(typeConversionToDouble(temperature.getValue()+""));
data.setCompressorStatus(typeConversionToInteger(compressorStatus.getValue()+""));
data.setResultCompressorStatus(resultCompressorStatus.getValue()+"");
data.setResultTemperature(resultTemperature.getValue()+"");
data.setResultHumidity(resultHumidity.getValue()+"");
data.setResultGroundingCurrent(resultGroundingCurrent.getValue()+"");
SimpleDateFormat sd = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sd.setTimeZone(TimeZone.getTimeZone("GMT+0"));//**TimeZone时区，加上这句话就解决啦**
data.setTime(sd.parse(keyAsString));
eeMonitorDataList.add(data);
}
return eeMonitorDataList;
}
private Double typeConversionToDouble (String str){
if("Infinity".equals(str)){
return Double.valueOf("0");
}else{
return Double.valueOf(str);
}
}
private Integer typeConversionToInteger (String str){
if("-Infinity".equals(str)){
return Integer.parseInt("0");
}else{
return Double.valueOf(String.valueOf(str)).intValue();
}
}
private SearchResponse getSearchResponse(String assetUid, Integer interval, Date startDate, Date endDate) throws IOException {
SearchRequest searchRequest = new SearchRequest("ee_monitor_data");
//构建查询
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolBuilder = QueryBuilders.boolQuery();
MatchQueryBuilder matchQueryBuilder1 = QueryBuilders.matchQuery("assetUid", assetUid);
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("time");
//起始时间
Calendar c = Calendar.getInstance();
c.setTime(startDate);
c.add(Calendar.HOUR_OF_DAY, -8);
Date newdateStartDate = c.getTime();
c.setTime(endDate);
c.add(Calendar.HOUR_OF_DAY, -8);
Date newdateEndDate = c.getTime();
SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd HH:mm");
String startDate2 = sdf2.format(newdateStartDate);
String endDate2 = sdf2.format(newdateEndDate);
StringBuilder sb = new StringBuilder(startDate2);
sb.setCharAt(10, 'T');
rangeQueryBuilder.gte(sb.toString());
//结束时间
//起始时间
StringBuilder sb2 = new StringBuilder(endDate2);
sb2.setCharAt(10, 'T');
rangeQueryBuilder.lte(sb2.toString());
boolBuilder.must(matchQueryBuilder1).must(rangeQueryBuilder);
sourceBuilder.query(boolBuilder);
//按时间聚合，求TX的和
//DateHistogramInterval.minutes(5)是指按5分钟聚合
//format("yyyy-MM-dd HH:mm")是指聚合的结果的Time的格式
//BucketOrder.aggregation("tx_sum", false)对聚合结果的排序 true为正序 false为倒序
AggregationBuilder aggregation = AggregationBuilders.dateHistogram("by_time").field("time").fixedInterval(DateHistogramInterval.minutes(interval)).format("yyyy-MM-dd HH:mm:ss")
.order(BucketOrder.aggregation("by_humidity", false));
aggregation.subAggregation(AggregationBuilders.sum("by_humidity").field("humidity"));//构建聚合条件字段
aggregation.subAggregation(AggregationBuilders.avg("by_humidity2").field("humidity"));//构建聚合条件字段
aggregation.subAggregation(AggregationBuilders.avg("by_groundingCurrent").field("groundingCurrent"));//构建聚合条件字段
aggregation.subAggregation(AggregationBuilders.avg("by_temperature").field("temperature"));//构建聚合条件字段
aggregation.subAggregation(AggregationBuilders.max("by_compressorStatus").field("compressorStatus"));//构建聚合条件字段
aggregation.subAggregation(AggregationBuilders.max("by_resultCompressorStatus").field("resultCompressorStatus"));//构建聚合条件字段
aggregation.subAggregation(AggregationBuilders.max("by_resultTemperature").field("resultTemperature"));//构建聚合条件字段
aggregation.subAggregation(AggregationBuilders.max("by_resultHumidity").field("resultHumidity"));//构建聚合条件字段
aggregation.subAggregation(AggregationBuilders.max("by_resultGroundingCurrent").field("resultGroundingCurrent"));//构建聚合条件字段
sourceBuilder.aggregation(aggregation);
searchRequest.source(sourceBuilder);
//发送请求
SearchResponse searchResponse = null;
searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
return searchResponse;
}
private Object[] getDateRangeBymode(Date startTime, Date endTime, Integer interval, String mode) {
Object[] results = null;
if ("byDay".equals(mode) || "byWeek".equals(mode) || "byMonth".equals(mode)) {
results = CaptureDataUtils.getDateRangeByMode(mode, interval);
} else {
if (startTime == null && endTime == null) {
results = CaptureDataUtils.getDateRangeByMode("byDay", interval);
}
}
return results;
}
public final String formatAsTwoDigits(Double input) {
if (input == null) {
return "";
}
return df.format(input);
}
public final <T> Object null2EmptyString(T obj) {
if (obj == null) {
return "";
}
return obj;
}
public String translate(String value){
switch (value){
case "1.0":
return "正常";
case "2.0":
return "报警";
default:
return "";
}
}
public static boolean judgeIsDecimal(String num){
boolean isdecimal = false;
if (num.contains(".")) {
isdecimal=true;
}
return isdecimal;
}
public List<Map<String, Object>> getSearchByAssetUids(List<String> assetUids) throws IOException{
List<Map<String, Object>> sourceList = new ArrayList();
Map<String, List<Map<String, Object>>> map = new HashMap<>();
int pageNum = 0;
while(pageNum < 5) {
SearchRequest searchRequest = new SearchRequest("ee_monitor_data");
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("time");
// rangeQueryBuilder.gte(getXHoursAgo(3));
BoolQueryBuilder childBoolQueryBuilder = new BoolQueryBuilder();
for (int j = 0; j < assetUids.size(); j++) {
childBoolQueryBuilder.should(QueryBuilders.matchQuery("assetUid", assetUids.get(j)));
}
boolQueryBuilder.must(childBoolQueryBuilder);
sourceBuilder.from(pageNum*2000);
sourceBuilder.size(2000);
sourceBuilder.query(boolQueryBuilder);
sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
sourceBuilder.sort(new FieldSortBuilder("time").order(SortOrder.DESC));
searchRequest.source(sourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
pageNum++;
for (SearchHit documentFields:searchResponse.getHits().getHits()) {
Map<String, Object> sourceAsMap = documentFields.getSourceAsMap();
if (!map.containsKey(sourceAsMap.get("assetUid"))) {//map中存在此id，将数据存放当前key的map中
List<Map<String, Object>> tmpList = new ArrayList<>();
tmpList.add(sourceAsMap);
map.put((String)sourceAsMap.get("assetUid"), tmpList);
}
}
if(map.keySet().size() >= assetUids.size()){
pageNum=5;
}
}
Set<String> keys = map.keySet();
for (String key : keys) {
sourceList.add(map.get(key).get(0));
}
return sourceList;
}
private String getXHoursAgo(int x) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Calendar c = Calendar.getInstance();
c.setTime(new Date());
c.add(Calendar.HOUR_OF_DAY, (-x-8));
StringBuilder sb = new StringBuilder(sdf.format(c.getTime()));
sb.setCharAt(10, 'T');
return sb.toString();
}
}
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
