API网关监控简单实现

一. 需求

API网关统计： 调用量，响应时间，响应码等等，时间粒度为分钟
(参考阿里云网关的一些参数)

二. 思路

1. 首先，框架为 Spring Cloud Gateway，通过路由配置filter进行统计
2. 各台主机统计请求的数据到本地内存，避免网络io影响接口
   • 使用LongAdder计数
   • 使用ConcurretHashMap保存各个接口的统计数据
3. 定时任务：每分钟各主机同步累加到redis
   • 首先使用一个redis set存放所有接口url
   • 每个接口url使用一个redis hash存放统计数据，key为时间(分钟)，value为统计数据
   • 各个主机通过分布式锁进行同步，保存到分布式缓存中
4. 定时任务：每小时从redis持久化到数据库

三. 实现

依赖：Redisson，spring-cloud-gateway，spring-webflux

1. filter&factory

/**
 * 监控过滤器
 * 1. 请求数
 * 2. 响应时间
 * 3. 状态码(失败数)
 * 4. 错误数(Api网关异常)
 */
@Slf4j
@Getter
@Component
public class MonitorFilter implements GatewayFilter, Ordered {

    private static final String START_TIME = "MonitorStartTime";

    @Autowired
    MonitorService monitorService;

    @Override
    public Mono<Void> filter(ServerWebExchange exchange, GatewayFilterChain chain) {

        String url = exchange.getRequest().getURI().getPath();

        MonitorService.MonitorParams params = monitorService.getMonitorParams(url);
        LongAdder reqNum = params.getReqNum();
        LongAdder res5xxNum = params.getRes5xxNum();
        LongAdder errorNum = params.getErrorNum();
        LongAdder resTime = params.getResTime();

        reqNum.increment();

        exchange.getAttributes().put(START_TIME, System.currentTimeMillis());
        return chain.filter(exchange).doOnError(e -> {
            //本地调用异常数
            errorNum.increment();
        }).then(
                Mono.fromRunnable(() -> {
                    Long startTime = exchange.getAttribute(START_TIME);
                    if (startTime != null) {
                        //相应时间累加
                        resTime.add(System.currentTimeMillis() - startTime);
                        //5xx服务器异常数
                        if (exchange.getResponse().getStatusCode().is5xxServerError()) {
                            res5xxNum.increment();
                        }
                    }
                })
        );
    }

    @Override
    public int getOrder() {
        return Ordered.HIGHEST_PRECEDENCE;
    }
}

@Component
public class MonitorGatewayFilterFactory extends AbstractGatewayFilterFactory {

    @Autowired
    MonitorFilter monitorFilter;

    public GatewayFilter apply() {
        return apply(o -> {
        });
    }

    @Override
    public GatewayFilter apply(Object config) {
        return monitorFilter;
    }
}

2. service

@Service
@Slf4j
public class MonitorService {


    private Map<String, MonitorParams> map = new ConcurrentHashMap<>();

    @Autowired
    RedissonClient redissonClient;

    @Autowired
    MonitorRecordMapper monitorRecordMapper;

    @Autowired
    MonitorUrlMapper monitorUrlMapper;


    public static final String LOCK_MONITOR_FLUSH = "lock_monitor_flush_";
    public static final String LOCK_MONITOR_SAVE = "lock_monitor_save_";
    public static final String CACHE_MONITOR = "cache_monitor";

    public MonitorParams getMonitorParams(String url) {
        MonitorParams params = map.getOrDefault(url, new MonitorParams());
        map.putIfAbsent(url, params);
        return params;
    }


    /**
     * 监控数据同步
     *
     * @param time
     */
    public void flush(String time) {


        log.info("监控数据开始同步：{}", time);

        Flux.fromIterable(map.entrySet())
                .publishOn(Schedulers.parallel())
                .subscribe(entry -> {
                    log.info("flush");


                    //url处理，去除尾部正斜杠(/)
                    String url = entry.getKey().replaceAll("/$", "");

                    //根据url加锁
                    String lockKey = LOCK_MONITOR_FLUSH + url;
                    RLock lock = redissonClient.getLock(lockKey);
                    try {
                        lock.lock();

                        TimeUnit.SECONDS.sleep(1);

                        //读取并重置监控数据
                        MonitorParams params = entry.getValue();
                        MonitorData data = new MonitorData(url, time,
                                params.getReqNum().sumThenReset(),
                                params.getRes5xxNum().sumThenReset(),
                                params.getErrorNum().sumThenReset(),
                                params.getResTime().sumThenReset());

                        //不在监控名单中不会保存
                        if (!isMonitorUrl(url)) {
                            return;
                        }

                        //如果监控数据为空，则不进行处理
                        if (data.getReqNum() == 0) {
                            return;
                        }

                        //根据url创建缓存
                        String cacheKey = String.format("%s_%s", CACHE_MONITOR, url);

                        //将缓存key加入缓存列表
                        RSet<String> cacheList = redissonClient.getSet(CACHE_MONITOR);
                        cacheList.add(cacheKey);

                        //将监控数据放入redis缓存
                        RMap<String, MonitorData> cacheMap = redissonClient.getMap(cacheKey);
                        MonitorData result = cacheMap.putIfAbsent(time, data);
                        if (result != null) {
                            //如果数据已存在则累加
                            cacheMap.put(time, data.add(result));
                        }

                    } catch (Exception e) {
                        log.error("监控数据同步失败：" + url, e);
                    } finally {
                        lock.unlock();
                    }
                });


        log.info("监控数据同步结束：{}", time);

    }


    /**
     * 监控数据入库
     *
     * @param time
     */
    public void save(String time) {

        RLock lock = redissonClient.getLock(LOCK_MONITOR_SAVE + time);
        //上锁失败则跳过
        if (!lock.tryLock()) {
            log.info("监控数据其他服务正在入库");
            return;
        }

        log.info("监控数据开始入库：{}", time);
        try {


            RSet<String> cacheList = redissonClient.getSet(CACHE_MONITOR);
            if (cacheList == null) {
                return;
            }

            Flux.fromIterable(cacheList)
                    .publishOn(Schedulers.newParallel("monitor_save"))
                    .subscribe(cacheKey -> {

                        RMap<String, MonitorData> map = redissonClient.getMap(cacheKey);
                        map.entrySet()
                                .forEach(entry -> {
                                    String key = entry.getKey();
                                    try {
                                        MonitorData value = entry.getValue();
                                        log.info("{}->{}", key, value);
                                        if (value != null) {
                                            //入库
                                            insertData(value);
                                            map.remove(key);
                                        }
                                    } catch (Exception e) {
                                        log.error(String.format("监控数据开始入库失败：%s-%s", cacheKey, key), e);
                                    }
                                });
                    });


        } finally {
            lock.unlock();
        }
        log.info("监控数据入库结束：{}", time);

    }

    private void insertData(MonitorData data) {
        MonitorRecord s = new MonitorRecord();
        s.setTime(Long.parseLong(data.getTime()));
        s.setUrl(data.getUrl());
        s.setReqNum(data.getReqNum());
        s.setErrorNum(data.getErrorNum());
        s.setRes5xxNum(data.getRes5xxNum());
        s.setResTime(data.getResTime());
        monitorRecordMapper.insertSelective(s);
    }

    /**
     * 检查是否为监控url
     *
     * @param url
     * @return
     */
    private boolean isMonitorUrl(String url) {
        List<MonitorUrl> list = getProxy().selectAllUrl();
        return list == null ? false : list.contains(url);
    }

    @Cacheable(value = "fulin-squeezy-cache", key = "allMonitorUrl")
    public List<MonitorUrl> selectAllUrl() {
        log.info("存储监控url缓存");
        return monitorUrlMapper.selectAll();
    }

    @CacheEvict(value = "fulin-squeezy-cache", key = "allMonitorUrl")
    public void delUrlCache() {
        log.info("移除监控url缓存");
    }


    private MonitorService getProxy() {
        try {
            return (MonitorService) AopContext.currentProxy();
        } catch (IllegalStateException e) {
            log.error("无代理实例", e);
            return this;
        }
    }

    public Mono<MonitorUrlInfo> create(Mono<MonitorUrlReq> just) {
        return just.flatMap(
                r -> {
                    MonitorUrl param = new MonitorUrl();
                    param.setUrl(r.getUrl());
                    if (monitorUrlMapper.selectOne(param) != null) {
                        throw new IllegalArgumentException("监控url已存在");
                    }
                    monitorUrlMapper.insertSelective(param);
                    getProxy().delUrlCache();
                    MonitorUrlInfo info = BeanCopyUtil.objConvert(param, MonitorUrlInfo.class);
                    return Mono.just(info);
                }
        );
    }

    public Mono<Long> del(Mono<Long> id) {
        return id.flatMap(
                i -> {
                    MonitorUrl monitorUrl = monitorUrlMapper.selectByPrimaryKey(id);
                    if (monitorUrl == null) {
                        throw new IllegalArgumentException("监控url不存在");
                    }

                    monitorUrlMapper.deleteByPrimaryKey(i);
                    getProxy().delUrlCache();
                    return Mono.just(i);
                }
        );
    }


    @Data
    @AllArgsConstructor
    @NoArgsConstructor
    public static class MonitorData {

        String url;
        String time;
        Long reqNum;
        Long res5xxNum;
        Long errorNum;
        Long resTime;

        public MonitorData add(MonitorData data) {
            reqNum += data.reqNum;
            res5xxNum += data.res5xxNum;
            errorNum += data.errorNum;
            resTime += data.resTime;
            return this;
        }
    }

    @Data
    public static class MonitorParams {
        //请求数
        private LongAdder reqNum = new LongAdder();
        //5xx响应数
        private LongAdder res5xxNum = new LongAdder();
        //调用错误数(Api网关异常)
        private LongAdder errorNum = new LongAdder();
        //响应时间
        private LongAdder resTime = new LongAdder();
    }
}