1. 使用例子
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Sort.Direction;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.aggregation.Aggregation;
import org.springframework.data.mongodb.core.aggregation.AggregationResults;
import org.springframework.data.mongodb.core.aggregation.GroupOperation;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSON;
import ai.winbonscloud.dto.FilterDTO;
import ai.winbonscloud.dto.PageDTO;
import ai.winbonscloud.dto.RequestDTO;
import ai.winbonscloud.dto.ResultDTO;
import ai.winbonscloud.dto.SortDTO;
import ai.winbonscloud.framework.exception.WinbonsCloudBizException;
import ai.winbonscloud.framework.utils.concurrent.threadpool.QueuableCachedThreadPool;
import ai.winbonscloud.framework.utils.concurrent.threadpool.ThreadPoolBuilder;
import ai.winbonscloud.framework.utils.concurrent.threadpool.ThreadPoolBuilder.QueuableCachedThreadPoolBuilder;
import ai.winbonscloud.framework.utils.io.IOUtil;
import ai.winbonscloud.framework.utils.time.DateFormatUtil;
import ai.winbonscloud.framework.utils.time.DateUtil;
import ai.winbonscloud.model.guidance.TimeUtil;
import ai.winbonscloud.mongo.log.RobotAudit;
import ai.winbonscloud.mongo.log.RobotAuditCount;
import ai.winbonscloud.mongo.log.RobotAuditPOJO;
import ai.winbonscloud.orm.id.sequence.SystemClock;
import ai.winbonscloud.orm.kit.Prop;
import ai.winbonscloud.orm.kit.PropKit;
import ai.winbonscloud.orm.plugin.activerecord.Db;
import ai.winbonscloud.orm.plugin.activerecord.Record;
import ai.winbonscloud.service.AbstractService;
/**
* Mongodb操作
*/
@Service
public class RobotAuditServiceImp extends AbstractService{
@Autowired
private MongoTemplate mongoTemplate;
/**
* 解析本地文件到mongodb数据库中
* @param requestDTO.getString(key)
* @return
*/
public ResultDTO<Object> save(RequestDTO requestDTO){
ResultDTO<Object> resultDTO = new ResultDTO<>();
File file = new File(requestDTO.getString("absolutePath"));
FileInputStream in =null;
FileOutputStream out = null;
BufferedReader reader = null;
boolean isDelete = true;
QueuableCachedThreadPoolBuilder queuableCachedPoolBuilder = ThreadPoolBuilder.queuableCachedPool();
QueuableCachedThreadPool threadPool = null;
try {//读取文件并解析保存到mongodb中
threadPool = queuableCachedPoolBuilder.build();
in = new FileInputStream(file);
reader = new BufferedReader(new InputStreamReader(in,"UTF-8"));
String len = null;
while((len = reader.readLine())!= null){
String data = len.substring(len.indexOf("["),len.lastIndexOf("}"));
//json转换为对应的实体，然后复制属性值到mongodb实体中进行保存
List<RobotAuditPOJO> parseList = JSON.parseArray(data, RobotAuditPOJO.class);
List<RobotAudit> temps = new ArrayList<>();
for (RobotAuditPOJO pojo : parseList) {
try {
//使用时间给个默认值，方便后面统计
if (pojo.getUsingTime() == null) {
pojo.setUsingTime(0L);
}
pojo.setIpAddress(requestDTO.getString("ipAddress"));
RobotAudit robotAudit = new RobotAudit();
//spring的beanUtils可以复制实体继承的属性
BeanUtils.copyProperties(pojo, robotAudit);
temps.add(robotAudit);
// 超过200条就开个线程处理一次 并清空临时集合
if (temps.size() >= 200) {
List<RobotAudit> robotAudits = new ArrayList<>(temps);
threadPool.execute(()->{ mongoTemplate.insert(robotAudits, RobotAudit.class);});
temps.clear();
}
}catch (Exception e) {
throw new WinbonsCloudBizException(e.getMessage());
}
}
threadPool.execute(()->{ mongoTemplate.insert(temps, RobotAudit.class);});
}
} catch (Exception e) {
logger.error("文件解析异常！"+e);
resultDTO.setSuccess(false);
resultDTO.setError("文件解析异常！"+e.getMessage());
//读取异常，保存该文档到异常文件夹
try {
Prop prop = PropKit.use("config/robotAudit.properties");
String errorDirectory = prop.get("errorDirectory");
//构建异常目录，二级目录是当天日期
File dest = new File(errorDirectory+File.separator+DateFormatUtil.formatDate(DateFormatUtil.PATTERN_ISO_ON_DATE, SystemClock.now())+File.separator+file.getName());
if (!dest.getParentFile().exists()) {
dest.getParentFile().mkdirs();
}
out = new FileOutputStream(dest);
IOUtils.copy(in, out);
} catch (IOException e1) {
logger.error("解析异常文件转移失败！"+e1);
//如果文件转移失败，那么就先不删除
isDelete = false;
}
}finally {//关流
if (reader!= null) {
IOUtil.closeQuietly(reader);
}
if (in != null) {
IOUtil.closeQuietly(in);
}
if (out != null ) {
IOUtil.closeQuietly(out);
}
//删除原始文件
if (isDelete) {
file.delete();
}
//关闭线程池
if (threadPool != null) {
threadPool.shutdown();
}
}
return resultDTO;
}
/**
* 分页查询日志记录
* @param requestDTO
* @return
*/
public ResultDTO<Object> getByPage(RequestDTO requestDTO){
ResultDTO<Object> resultDTO = new ResultDTO<>();
Map<String, Object> result = new HashMap<String, Object>();
try {
Query query = new Query();
//这里根据前端的过滤条件进行条件筛选
List<FilterDTO> filters = requestDTO.getFilters();
if (filters != null && filters.size()>0) {
Criteria criteria = null;
for (FilterDTO filter : filters) {
if ("=".equals(filter.getOperation())) {
//如果是租户的查询那么需要查询租户和他下面的租户
if ("tenantId".equals(filter.getProperty())) {
//如果是华邦云默认查询全部
if (10000 != (Integer)filter.getValue()) {
List<Long> tenantIds = new ArrayList<Long>();//getTenantIds(filter);
query.addCriteria(Criteria.where(filter.getProperty()).in(tenantIds));
}
}else{
query.addCriteria(Criteria.where(filter.getProperty()).is(filter.getValue()));
}
}
if (">=".equals(filter.getOperation())) {
criteria = Criteria.where(filter.getProperty()).gte(filter.getValue());
}
if ("<=".equals(filter.getOperation())) {
if (criteria != null) {
criteria.andOperator(Criteria.where(filter.getProperty()).lte(filter.getValue()));
}else{
criteria = Criteria.where(filter.getProperty()).lte(filter.getValue());
}
}
}
if(criteria != null){
query.addCriteria(criteria);
}
}
//设置排序
List<SortDTO> sorts = requestDTO.getSorts();
if (sorts != null && sorts.size()>0 ) {
sorts.forEach(p->{
if ("desc".equals(p.getOrder())) {
query.with(new Sort(Direction.DESC, p.getProperty()));
}else{
query.with(new Sort(Direction.ASC, p.getProperty()));
}
});
}else{//默认是按照时间的降序排序
query.with(new Sort(Direction.DESC, "id"));
}
//设置分页
PageDTO page = requestDTO.getPage();
if (page == null) {
page = new PageDTO();
}
int limit = page.getLimit();
query.skip(page.getFirstResult());
query.limit(limit);
List<RobotAudit> findAll = mongoTemplate.find(query,RobotAudit.class, "RobotAudit");
//从数据库获取更详细的信息
List<Map<String, Object>> items = new ArrayList<Map<String, Object>>();//convertToDetailRecord(findAll);
result.put("items", items);
long count = mongoTemplate.count(query,RobotAudit.class,"RobotAudit");
long totalPages = count%limit== 0 ? count/limit : count/limit+1;
if (totalPages == 0 ) {
totalPages = 1;
}
result.put("totalPages", totalPages);
result.put("totalCount", count);
resultDTO.setData(result);
} catch (Exception e) {
e.printStackTrace();
logger.error(e.getMessage());
}
return resultDTO;
}
public ResultDTO<Object> robotAuditFilter(RequestDTO requestDTO) {
/*
* 1.先读取mysql数据库获取当前日志处理的进度（时间戳为标记）
* 2.然后读取mongodb大于该时间戳的日志进行过滤转换
* 3.在结束的时候吧最后那条记录的时间戳保存到mysql表中以便下次处理
* */
Long startTime = null;
try {
Record record = Db.findFirst(null, "SELECT * FROM s_robotaudit_flag t WHERE t.lastFilterTime = (SELECT MAX(lastFilterTime) FROM s_robotaudit_flag)");
if (record == null) {//初始化开始时间为：上线前一天 17-6-27
Record flag = new Record();
flag.set("lastFilterTime", TimeUtil.parseDate("2017-6-27").getTime());
flag.set("createDate", SystemClock.now());
Db.save(null, "s_robotaudit_flag",flag);
//再查一次
record = Db.findFirst(null, "SELECT * FROM s_robotaudit_flag");
}
if (record != null) {
Long lastFilterTime = record.getLong("lastFilterTime");
startTime = lastFilterTime;
//查询日志详情表中是否还有数据
Long count = countByday(lastFilterTime);
while(count != null && count > 0){
//查询上次结束时间（设置为第二天0点）开始的一天的数据 ，既 当天0点 <= x < 第二天0点
//当天0点
Date beginOfDate = DateUtil.beginOfDate(new Date(lastFilterTime));
//第二天0点
Date startDate = DateUtil.addDays(beginOfDate, 1);
Date endDate = DateUtil.addDays(startDate, 1);
//查询mongo
Criteria criteria = Criteria.where("sessionTime").gte(startDate.getTime()).andOperator(Criteria.where("sessionTime").lt(endDate.getTime()));
Query query = new Query();
query.addCriteria(criteria);
query.with(new Sort(Direction.DESC, "sessionTime"));
RobotAudit findTheLastOne = mongoTemplate.findOne(query, RobotAudit.class, "RobotAudit");
if (findTheLastOne == null) {
lastFilterTime += 24*3600000; //没记录就加一天
count = countByday(lastFilterTime);
continue;
}
//多条件分组统计查询不同租户下不同机器的服务使用次数和使用时长
AggregationResults<Map> results = groupQuery(criteria,"tenantId","machineCode","serviceId");
//保存统计日志
Integer failureCount = saveRobotAuditCount(startDate, results);
//当天总共多少条
Long countOfToday = mongoTemplate.count(query, RobotAudit.class, "RobotAudit");
//更新当天最后处理那条记录的时间到mysql数据库中
lastFilterTime = findTheLastOne.getSessionTime();
Record flag = new Record();
flag.set("lastFilterTime", lastFilterTime);
flag.set("createDate", SystemClock.now());
flag.set("totalCount", countOfToday);
flag.set("failureCount", failureCount);
Db.save(null, "s_robotaudit_flag",flag);
//查询mongdb还有数据待处理否
count = countByday(lastFilterTime);
logger.info("【当天处理结束时的会话时间是："+lastFilterTime+"】");
}
}
} catch (Exception e) {
logger.error("【日志洗刷异常！】,开始时间是："+startTime+"】",e);
}
return new ResultDTO<>();
}
//保存统计日志
private Integer saveRobotAuditCount(Date startDate, AggregationResults<Map> results) {
Integer failureCount = null;
//遍历统计结果，添加到日志统计表
for (Map map : results) {
try {
RobotAuditCount robotAuditCount = new RobotAuditCount();
ai.winbonscloud.framework.utils.bean.BeanUtils.convertMapToObj(map, robotAuditCount);
robotAuditCount.setSessionTimeOfDay(startDate.getTime());
mongoTemplate.save(robotAuditCount);
} catch (Exception e) {
logger.error("【一条统计日志保存失败】",e);
failureCount = (Integer)map.get("usingCount");
}
}
return failureCount;
}
//mongodb分组统计查询日志
private AggregationResults<Map> groupQuery(Criteria criteria,String...filed) {
GroupOperation group = Aggregation.group(filed).count().as("usingCount").sum("usingTime").as("usingTime");
Aggregation agg = Aggregation.newAggregation(RobotAudit.class,Aggregation.match(criteria) ,group);
AggregationResults<Map> aggregate = mongoTemplate.aggregate(agg,"RobotAudit",Map.class);
return aggregate;
}
//统计大于改时间的日志记录数
private Long countByday(Long lastFilterTime) {
Query query = new Query();
query.addCriteria(Criteria.where("sessionTime").gte(lastFilterTime));
Long count= mongoTemplate.count(query,RobotAudit.class,"RobotAudit");
return count;
}
public ResultDTO<Object> handleAbnormalRobotAuditCount(RequestDTO requestDTO) {
ResultDTO<Object> resultDTO = new ResultDTO<>();
try {
//查询数据库记录的有失败记录的那天日志记录
List<Record> fialureRecords = Db.find(null, "SELECT * FROM s_robotaudit_flag t WHERE t.failureCount IS NOT NULL AND t.failureCount != 0");
if (fialureRecords != null && fialureRecords.size() > 0) {
for (Record record : fialureRecords) {
try {
//获取那天的时间。然后重新过滤一下那天的日志
Long lastFilterTime = record.getLong("lastFilterTime");
//获取当天0点。
Date startDate = DateUtil.beginOfDate(new Date(lastFilterTime));
//第二天0点
Date endDate = DateUtil.addDays(startDate, 1);
Criteria criteria = Criteria.where("sessionTime").gte(startDate.getTime()).andOperator(Criteria.where("sessionTime").lt(endDate.getTime()));
AggregationResults<Map> results = groupQuery(criteria,"tenantId","machineCode","serviceId");
Query query = new Query();
query.addCriteria(Criteria.where("sessionTimeOfDay").is(startDate.getTime()));
//删除当天日志
mongoTemplate.remove(query, RobotAuditCount.class, "RobotAuditCount");
//重新统计结果并保存
Integer failureCount = saveRobotAuditCount(startDate, results);
//更新数据库那条记录的失败数
Db.update(null, "UPDATE s_robotaudit_flag SET failureCount = ? WHERE id = ? ", failureCount,record.getLong("id"));
} catch (Exception e) {
e.printStackTrace();
}
}
}
} catch (Exception e) {
resultDTO.setError("处理异常日志失败！");
resultDTO.setSuccess(false);
}
return resultDTO;
}
}
2.Model实体映射
import org.springframework.data.mongodb.core.mapping.Document;
@Document(collection = "UserCount")
public class UserCount implements java.io.Serializable {
private static final long serialVersionUID = 1L;
/**
* 使用次数
*/
private Integer usingCount;
/**
* 使用时间
*/
private Long usingTime;
public Integer getUsingCount() {
return usingCount;
}
public void setUsingCount(Integer usingCount) {
this.usingCount = usingCount;
}
public Long getUsingTime() {
return usingTime;
}
public void setUsingTime(Long usingTime) {
this.usingTime = usingTime;
}
}
3.配置文件连接Mongodb库
application.properties :
#=========== MongoDB Begin ==========
spring.data.mongodb.host=192.168.2.166
spring.data.mongodb.port=30000
spring.data.mongodb.database=ai_CONSOLE
#=========== MongoDB End ==========
怎加Maven仓库支持 :
<!-- 增加mongodb支持 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
扫一扫
2918
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
