【网站点击流实时数据分析】05-同步程序实现

1、程序目录

2、callback

/**
 * Describe: 计算每天的全量数据
 */
public class DayAppendCallBack implements Runnable{
    @Override
    public void run() {
        Calendar calendar = Calendar.getInstance();
        if (calendar.get(Calendar.MINUTE) == 0 && calendar.get(Calendar.HOUR) == 0) {
            String endTime = DateUtils.getDataTime(calendar);
            String startTime = DateUtils.beforeOneDay(calendar);
            LogAnalyzeDao logAnalyzeDao = new LogAnalyzeDao();
            List<BaseRecord> baseRecordList = logAnalyzeDao.sumRecordValue(startTime, endTime);
            logAnalyzeDao.saveDayAppendRecord(baseRecordList);
        }
    }
}

/**
 * Describe: 计算每半个小时的数据
 */
public class HalfAppendCallBack implements Runnable {
    @Override
    public void run() {
        Calendar calendar = Calendar.getInstance();
        if (calendar.get(Calendar.MINUTE) % 30 == 0) {
            String endTime = DateUtils.getDataTime(calendar);
            String startTime = DateUtils.before30Minute(calendar);
            LogAnalyzeDao logAnalyzeDao = new LogAnalyzeDao();
            List<BaseRecord> baseRecordList = logAnalyzeDao.sumRecordValue(startTime, endTime);
            logAnalyzeDao.saveHalfAppendRecord(baseRecordList);
        }
    }
}

/**
 * Describe: 计算每小时的增量数据
 */
public class HourAppendCallBack implements Runnable {
    @Override
    public void run()  {
        Calendar calendar = Calendar.getInstance();
        System.out.println(calendar.get(Calendar.MINUTE));
        if (calendar.get(Calendar.MINUTE) == 59) {
            //获取所有的增量数据
            String endTime = DateUtils.getDataTime(calendar);
            String startTime = DateUtils.beforeOneHour(calendar);
            LogAnalyzeDao logAnalyzeDao = new LogAnalyzeDao();
            List<BaseRecord> baseRecordList = logAnalyzeDao.sumRecordValue(startTime, endTime);
            logAnalyzeDao.saveHourAppendRecord(baseRecordList);
        }
    }
}

/**
 * Describe: 计算每分钟的增量信息
 */
public class OneMinuteCallBack implements Runnable {
    @Override
    public void run() {
        Calendar calendar = Calendar.getInstance();
        //24:00分时，将缓存清空
        if (calendar.get(Calendar.MINUTE) == 0 && calendar.get(Calendar.HOUR) == 24) {
            CacheData.setPvMap(new HashMap<String, Integer>());
            CacheData.setUvMap(new HashMap<String, Long>());
        }
        String date = DateUtils.getDate();
        //从redis中获取所有的指标最新的值
        List<BaseRecord> baseRecordList = getBaseRecords(date);
        //根据cacheData获取所有的指标的增量值  用最新全量值减去上一个时间段的全量值
        List<BaseRecord> appendDataList = getAppData(baseRecordList);
        //将增量数据保存到mysql中
        new LogAnalyzeDao().saveMinuteAppendRecord(appendDataList);
    }

    private List<BaseRecord> getAppData(List<BaseRecord> baseRecordList) {
        List<BaseRecord> appendDataList = new ArrayList<BaseRecord>();
        for (BaseRecord baseRecord : baseRecordList) {
            //用最新的pv减去缓存中的pv值，得到最新的增量数据
            int pvAppendValue = CacheData.getPv(baseRecord.getPv(), baseRecord.getIndexName());
            //用最新的pv减去缓存中的pv值，得到最新的增量数据
            long uvAppendValue = CacheData.getUv(baseRecord.getUv(), baseRecord.getIndexName());
            appendDataList.add(new BaseRecord(baseRecord.getIndexName(), pvAppendValue, uvAppendValue, baseRecord.getProcessTime()));
        }
        return appendDataList;
    }

    private List<BaseRecord> getBaseRecords(String date) {
        List<LogAnalyzeJob> logAnalyzeJobList = new LogAnalyzeDao().loadJobList();
        ShardedJedis jedis = MyShardedJedisPool.getShardedJedisPool().getResource();
        List<BaseRecord> baseRecords = new ArrayList<>();
        for (LogAnalyzeJob analyzeJob : logAnalyzeJobList) {
            String pvKey = "log:" + analyzeJob.getJobName() + ":pv:" + date;
            String uvKey = "log:" + analyzeJob.getJobName() + ":uv:" + date;
            String pv = jedis.get(pvKey);
            long uv = jedis.scard(uvKey);
            BaseRecord baseRecord = new BaseRecord(analyzeJob.getJobName(), Integer.parseInt(pv.trim()), uv, new Date());
            baseRecords.add(baseRecord);
        }
        return baseRecords;
    }
}

3、dao

/**
 * Describe: 缓存上一分钟的数据，用来做cache
 * 整点的时候，cache层中的数据，需要清理掉
 */
public class CacheData {
    private static Map<String, Integer> pvMap = new HashMap<>();
    private static Map<String, Long> uvMap = new HashMap<>();

    public static int getPv(int pv, String indexName) {
        Integer cacheValue = pvMap.get(indexName);

        if (cacheValue == null) {
            cacheValue = 0;
            pvMap.put(indexName, cacheValue);
        }
        if (pv > cacheValue.intValue()) {
            pvMap.put(indexName, pv); //将新的值赋值个cacheData
            return pv - cacheValue.intValue();
        }
        return 0;//如果新的值小于旧的值，直接返回0
    }

    public static long getUv(long uv, String indexName) {
        Long cacheValue = uvMap.get(indexName);
        if (cacheValue == null) {
            cacheValue = 0l;
            uvMap.put(indexName, cacheValue);
        }
        if (uv > cacheValue.longValue()) {
            uvMap.put(indexName, uv);//将新的值赋值给cachaData
            return uv - cacheValue;
        }
        return 0;//如果新的值小于旧的值，直接返回0
    }

    public static Map<String, Integer> getPvMap() {
        return pvMap;
    }

    public static void setPvMap(Map<String, Integer> pvMap) {
        CacheData.pvMap = pvMap;
    }

    public static Map<String, Long> getUvMap() {
        return uvMap;
    }

    public static void setUvMap(Map<String, Long> uvMap) {
        CacheData.uvMap = uvMap;
    }
}

4、domain

public class BaseRecord {
    private String indexName;
    private String redisKey;
    private int pv;
    private long uv;
    private Date processTime;
}

5、启动类

/**
 * Describe: 计算每个指标每分钟的增量数据
 */
public class ReportDataWorker {

    public static void main(String[] args) {
        ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(10);
        //计算每分钟的增量数据，并将增量数据保存到mysql数据库中
        scheduledExecutorService.scheduleAtFixedRate(new OneMinuteCallBack(), 0, 60, TimeUnit.SECONDS);
        //计算每半个小时的增量数据，并将增量数据保存到mysql数据库中
        scheduledExecutorService.scheduleAtFixedRate(new HalfAppendCallBack(), 0, 60, TimeUnit.SECONDS);
        //计算每小时的增量数据，并将增量数据保存到mysql数据库中
        scheduledExecutorService.scheduleAtFixedRate(new HourAppendCallBack(), 0, 60, TimeUnit.SECONDS);
        //计算每天的全量，并将增量数据保存到mysql数据库中
        scheduledExecutorService.scheduleAtFixedRate(new DayAppendCallBack(), 0, 60, TimeUnit.SECONDS);
    }

}