引入Maven依赖:
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
</dependencies>
添加ES配置
spring:
elasticsearch:
rest:
uris: http://****:9200
username: elastic
password: 123456
data:
elasticsearch:
repositories:
enabled: true
cluster-nodes: 10.0.5.135:9300
cluster-name: elastic
cluster-password: 123456
代码
全文检索实体类
@Data
@ApiModel("地名地址")
@Document(indexName = "#{@elasticSearchConfig.elasticIndexPerf}"+"address", useServerConfiguration = true)
public class EsAddress implements Serializable {
@ApiModelProperty(value = "主键")
@Id
private String id;
@ApiModelProperty(value = "地名")
private String name;
@ApiModelProperty(value = "地址信息")
private String adress;
@ApiModelProperty(value = "地名类别")
private String type;
@ApiModelProperty(value = "户室号")
private String room;
}
Es索引前缀
@Configuration
@Data
public class ElasticSearchConfig {
@Value("${elastic_index_perf}")
public String elasticIndexPerf = "spatial_";
}
Es全文检索继承ElasticsearchRepository类
public interface AddressRepository extends ElasticsearchRepository<EsAddress, String> {
}
ES增删改查
@Autowired
private AddressRepository addressRepository;
public void realTimeSync(List<AdressDmpManageVO> managedHouseList) {
List<EsAddress> result = new ArrayList<>();
if (CollectionUtil.isNotEmpty(managedHouseList)) {
for (AdressDmpManageVO vo : managedHouseList) {
EsAddress es = new EsAddress();
BeanUtils.copyProperties(vo, es);
result.add(es);
}
try {
addressRepository.saveAll(result);
} catch (Exception exception) {
if (!(exception.getMessage()).contains("200 OK")) {
throw exception;
}
}
}
}
public void udpateToEs(AdressDmpManageVO vo) {
try {
if (addressRepository.existsById(vo.getId())) {
EsAddress es = new EsAddress();
BeanUtils.copyProperties(vo, es);
addressRepository.save(es);
}
} catch (Exception exception) {
if (!(exception.getMessage()).contains("200 OK")) {
throw exception;
}
}
}
public void deleteByAddressId(String addressId) {
try {
if (addressRepository.existsById(addressId)) {
addressRepository.deleteById(addressId);
}
} catch (Exception exception) {
if (!(exception.getMessage()).contains("200 OK")) {
throw exception;
}
}
}
public void deleteAll() {
new Thread(() -> {
try {
addressRepository.deleteAll();
} catch (Exception exception) {
if (!(exception.getMessage()).contains("200 OK")) {
throw exception;
}
}
}).start();
}
数据库数据同步到ES库
@GetMapping("/sync-address-list")
@ApiOperationSupport(order = 1)
@ApiOperation("数据_同步到ES")
public R syncHouseList() {
String cacheKey = elasticSearchConfig.getElasticIndexPerf() + "address";
boolean isRunCache = iDataCahceLogService.isRunCache(cacheKey);
if (!isRunCache) {
try {
String title = "数据";
boolean isRestartCache = iDataCahceLogService.startOrRestartCache(cacheKey, title, DataCacheLogEnum.DATA_CACHE_TYPE_ES.key);
new Thread(() -> {
esAddressService.importHouse(isRestartCache, cacheKey);
}).start();
} catch (Exception e) {
e.printStackTrace();
return R.fail("数据同步失败...");
}
return R.success("数据正在同步中,请稍后...");
} else {
return R.success("数据开始同步...");
}
}
@Override
@Transactional(rollbackFor = Exception.class)
public boolean startOrRestartCache(String hashKey, String title, String type) {
boolean reslut = false;
String id = null;
DataCacheLog lastTimeCache = getLastTimeCache(hashKey);
if (lastTimeCache != null) {
if (DataCacheLogEnum.DATA_CACHE_STATUS_STOP.key.equals(lastTimeCache.getStatus())) {
id = lastTimeCache.getId();
reslut = true;
} else if (DataCacheLogEnum.DATA_CACHE_STATUS_FAILED.key.equals(lastTimeCache.getStatus())) {
reslut = true;
}
}
DataCacheLog dataCahceLog = new DataCacheLog();
dataCahceLog.setKey(hashKey);
dataCahceLog.setStartTime(new Date());
dataCahceLog.setTitle(title);
dataCahceLog.setType(type);
dataCahceLog.setStatus(DataCacheLogEnum.DATA_CACHE_STATUS_RUN.key);
if (StringUtil.isNotBlank(id)) {
dataCahceLog.setId(id);
this.updateById(dataCahceLog);
} else {
dataCahceLog.setId(UUID.get32UUID());
this.save(dataCahceLog);
}
return reslut;
}
@Override
public void importHouse(boolean isRestartCache, String titleKey) {
LambdaQueryWrapper<AdressDmpManage> wrapper = new QueryWrapper().lambda();
long houseTotal = adressDmpManageService.count(wrapper);
long pageSize = 2000;
int threadSize = 10;
List<CountThreadPage> threadList = CalculationUtil.countThreadPage(houseTotal, pageSize, threadSize);
List<Future<?>> futures = new ArrayList<>();
try {
for (int i = 0; i < threadList.size(); i++) {
CountThreadPage threadPage = threadList.get(i);
futures.add(
executorService.submit(() -> {
long startPage = threadPage.getStartPage();
long endPage = threadPage.getEndPage();
for (long j = startPage; j <= endPage; j++) {
if (isRestartCache) {
boolean hasCacheRecord = iDataCacheRecordService.hasCacheRecord(titleKey, titleKey, j + "");
if (hasCacheRecord) {
continue;
}
}
List<EsAddress> esAddressList = syncListByPage(j, pageSize);
boolean isStopCache = iDataCahceLogService.isStopCache(titleKey);
if (isStopCache) {
executorService.shutdown();
break;
}
if (CollectionUtils.isNotEmpty(esAddressList)) {
try {
elasticsearchRestTemplate.save(esAddressList);
} catch (Exception exception) {
if (!(exception.getMessage()).contains("200 OK")) {
throw exception;
}
}
log.info("[elastic-house-delta] {} 分页数据_全量同步完成,当前页:{},大小:{}", Thread.currentThread().getName(), j, esAddressList.size());
iDataCacheRecordService.addCacheRecord(titleKey, titleKey, j + "");
}
}
})
);
}
for (Future<?> future : futures) {
future.get();
}
completeSync(titleKey);
log.info("===================数据列表_同步完成=========================");
} catch (Exception e) {
e.printStackTrace();
log.info("===================数据列表_同步失败=========================");
iDataCahceLogService.cacheFailed(titleKey, e.toString());
} finally {
executorService.shutdown();
}
}
public static List<CountThreadPage> countThreadPage(long total, long pageSize,long threadSize){
long totalPage = total/pageSize;
if (total % pageSize != 0) {
totalPage++;
}
return countThreadPage(totalPage, threadSize);
}
@Override
@Transactional(rollbackFor = Exception.class)
public void addCacheRecord(String titleKey, String cacheKey, String value) {
DataCacheRecord entity = new DataCacheRecord();
entity.setId(UUID.get32UUID());
entity.setTitleKey(titleKey);
entity.setCacheKey(cacheKey);
entity.setCacheValue(value);
entity.setCacheTime(new Date());
this.save(entity);
}
@Override
public boolean hasCacheRecord(String titleKey, String cacheKey, String value) {
boolean result = false;
LambdaQueryWrapper<DataCacheRecord> wrapper = new QueryWrapper().lambda();
wrapper.eq(DataCacheRecord::getTitleKey, titleKey);
wrapper.eq(DataCacheRecord::getCacheKey, cacheKey);
wrapper.eq(DataCacheRecord::getCacheValue, value);
long count = this.count(wrapper);
if (count > 0) {
result = true;
}
return result;
}
@Override
public boolean isStopCache(String key) {
boolean result = false;
DataCacheLog dataCacheLog = getLastTimeCache(key);
if (dataCacheLog != null && !DataCacheLogEnum.DATA_CACHE_STATUS_RUN.key.equals(dataCacheLog.getStatus())) {
result = true;
}
return result;
}