9.网站数据统计

1.Redis 高级数据类型

（1）HyperLogLog

统计20万个重复数据的独立总数

// 统计20万个重复数据的独立总数.
@Test
public void testHyperLogLog() {
    String redisKey = "test:hll:01";

    for (int i = 1; i <= 100000; i++) {
        redisTemplate.opsForHyperLogLog().add(redisKey, i);
    }

    for (int i = 1; i <= 100000; i++) {
        int r = (int) (Math.random() * 100000 + 1);
        redisTemplate.opsForHyperLogLog().add(redisKey, r);
    }

    long size = redisTemplate.opsForHyperLogLog().size(redisKey);
    System.out.println(size);
}

将3组数据合并, 再统计合并后的重复数据的独立总数

@Test
public void testHyperLogLogUnion() {
    String redisKey2 = "test:hll:02";
    for (int i = 1; i <= 10000; i++) {
        redisTemplate.opsForHyperLogLog().add(redisKey2, i);
    }

    String redisKey3 = "test:hll:03";
    for (int i = 5001; i <= 15000; i++) {
        redisTemplate.opsForHyperLogLog().add(redisKey3, i);
    }

    String redisKey4 = "test:hll:04";
    for (int i = 10001; i <= 20000; i++) {
        redisTemplate.opsForHyperLogLog().add(redisKey4, i);
    }

    String unionKey = "test:hll:union";
    redisTemplate.opsForHyperLogLog().union(unionKey, redisKey2, redisKey3, redisKey4);

    long size = redisTemplate.opsForHyperLogLog().size(unionKey);
    System.out.println(size);
}

（2）bitmap

统计一组数据的布尔值

@Test
public void testBitMap() {
    String redisKey = "test:bm:01";

    // 记录
    redisTemplate.opsForValue().setBit(redisKey, 1, true);
    redisTemplate.opsForValue().setBit(redisKey, 4, true);
    redisTemplate.opsForValue().setBit(redisKey, 7, true);

    // 查询
    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 0));
    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 1));
    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 2));

    // 统计
    Object obj = redisTemplate.execute(new RedisCallback() {
        @Override
        public Object doInRedis(RedisConnection connection) throws DataAccessException {
            return connection.bitCount(redisKey.getBytes());
        }
    });

    System.out.println(obj);
}

统计3组数据的布尔值, 并对这3组数据做OR运算

// 统计3组数据的布尔值, 并对这3组数据做OR运算.
@Test
public void testBitMapOperation() {
    String redisKey2 = "test:bm:02";
    redisTemplate.opsForValue().setBit(redisKey2, 0, true);
    redisTemplate.opsForValue().setBit(redisKey2, 1, true);
    redisTemplate.opsForValue().setBit(redisKey2, 2, true);

    String redisKey3 = "test:bm:03";
    redisTemplate.opsForValue().setBit(redisKey3, 2, true);
    redisTemplate.opsForValue().setBit(redisKey3, 3, true);
    redisTemplate.opsForValue().setBit(redisKey3, 4, true);

    String redisKey4 = "test:bm:04";
    redisTemplate.opsForValue().setBit(redisKey4, 4, true);
    redisTemplate.opsForValue().setBit(redisKey4, 5, true);
    redisTemplate.opsForValue().setBit(redisKey4, 6, true);

    String redisKey = "test:bm:or";
    Object obj = redisTemplate.execute(new RedisCallback() {
        @Override
        public Object doInRedis(RedisConnection connection) throws DataAccessException {
            connection.bitOp(RedisStringCommands.BitOperation.OR,
                    redisKey.getBytes(), redisKey2.getBytes(), redisKey3.getBytes(), redisKey4.getBytes());
            return connection.bitCount(redisKey.getBytes());
        }
    });

    System.out.println(obj);

    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 0));
    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 1));
    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 2));
    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 3));
    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 4));
    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 5));
    System.out.println(redisTemplate.opsForValue().getBit(redisKey, 6));
}

2.网站数据统计

仅有管理员有权限

（1）RedisKeyUtil

private static final String PREFIX_DAU = "dau";
private static final String PREFIX_POST = "post";

// 单日UV
public static String getUVKey(String date) {
    return PREFIX_UV + SPLIT + date;
}

// 区间UV
public static String getUVKey(String startDate, String endDate) {
    return PREFIX_UV + SPLIT + startDate + SPLIT + endDate;
}

// 单日活跃用户
public static String getDAUKey(String date) {
    return PREFIX_DAU + SPLIT + date;
}

// 区间活跃用户
public static String getDAUKey(String startDate, String endDate) {
    return PREFIX_DAU + SPLIT + startDate + SPLIT + endDate;
}

（2） DataService

@Service
public class DataService {

    @Autowired
    private RedisTemplate redisTemplate;

    private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd");

    // 将指定的IP计入UV
    public void recordUV(String ip) {
        String redisKey = RedisKeyUtil.getUVKey(df.format(new Date()));
        redisTemplate.opsForHyperLogLog().add(redisKey, ip);
    }

    // 统计指定日期范围内的UV
    public long calculateUV(Date start, Date end) {
        if (start == null || end == null) {
            throw new IllegalArgumentException("参数不能为空!");
        }

        // 整理该日期范围内的key
        List<String> keyList = new ArrayList<>();
        Calendar calendar = Calendar.getInstance();
        calendar.setTime(start);
        while (!calendar.getTime().after(end)) {
            String key = RedisKeyUtil.getUVKey(df.format(calendar.getTime()));
            keyList.add(key);
            calendar.add(Calendar.DATE, 1);
        }

        // 合并这些数据
        String redisKey = RedisKeyUtil.getUVKey(df.format(start), df.format(end));
        redisTemplate.opsForHyperLogLog().union(redisKey, keyList.toArray());

        // 返回统计的结果
        return redisTemplate.opsForHyperLogLog().size(redisKey);
    }

    // 将指定用户计入DAU
    public void recordDAU(int userId) {
        String redisKey = RedisKeyUtil.getDAUKey(df.format(new Date()));
        redisTemplate.opsForValue().setBit(redisKey, userId, true);
    }

    // 统计指定日期范围内的DAU
    public long calculateDAU(Date start, Date end) {
        if (start == null || end == null) {
            throw new IllegalArgumentException("参数不能为空!");
        }

        // 整理该日期范围内的key
        List<byte[]> keyList = new ArrayList<>();
        Calendar calendar = Calendar.getInstance();
        calendar.setTime(start);
        while (!calendar.getTime().after(end)) {
            String key = RedisKeyUtil.getDAUKey(df.format(calendar.getTime()));
            keyList.add(key.getBytes());
            calendar.add(Calendar.DATE, 1);
        }

        // 进行OR运算
        return (long) redisTemplate.execute(new RedisCallback() {
            @Override
            public Object doInRedis(RedisConnection connection) throws DataAccessException {
                String redisKey = RedisKeyUtil.getDAUKey(df.format(start), df.format(end));
                connection.bitOp(RedisStringCommands.BitOperation.OR,
                        redisKey.getBytes(), keyList.toArray(new byte[0][0]));
                return connection.bitCount(redisKey.getBytes());
            }
        });
    }
}

（3） DataInterceptor

每次请求都需要记录，计入拦截器比较合适

@Component
public class DataInterceptor implements HandlerInterceptor {

    @Autowired
    private DataService dataService;

    @Autowired
    private HostHolder hostHolder;

    @Override
    public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
        // 统计UV
        String ip = request.getRemoteHost();
        dataService.recordUV(ip);

        // 统计DAU
        User user = hostHolder.getUser();
        if (user != null) {
            dataService.recordDAU(user.getId());
        }

        return true;
    }
}

（4）WebMvcConfig

配置拦截器

registry.addInterceptor(dataInterceptor)
                .excludePathPatterns("/**/*.css", "/**/*.js", "/**/*.png", "/**/*.jpg", "/**/*.jpeg");

（5）DataController

@Controller
public class DataController {

    @Autowired
    private DataService dataService;

    // 统计页面
    @RequestMapping(path = "/data", method = {RequestMethod.GET, RequestMethod.POST})
    public String getDataPage() {
        return "/site/admin/data";
    }

    // 统计网站UV
    @RequestMapping(path = "/data/uv", method = RequestMethod.POST)
    public String getUV(@DateTimeFormat(pattern = "yyyy-MM-dd") Date start,
                        @DateTimeFormat(pattern = "yyyy-MM-dd") Date end, Model model) {
        long uv = dataService.calculateUV(start, end);
        model.addAttribute("uvResult", uv);
        model.addAttribute("uvStartDate", start);
        model.addAttribute("uvEndDate", end);
        return "forward:/data";
    }

    // 统计活跃用户
    @RequestMapping(path = "/data/dau", method = RequestMethod.POST)
    public String getDAU(@DateTimeFormat(pattern = "yyyy-MM-dd") Date start,
                         @DateTimeFormat(pattern = "yyyy-MM-dd") Date end, Model model) {
        long dau = dataService.calculateDAU(start, end);
        model.addAttribute("dauResult", dau);
        model.addAttribute("dauStartDate", start);
        model.addAttribute("dauEndDate", end);
        return "forward:/data";
    }
}

（6）SecurityConfig

管理员才能访问

.antMatchers(
        "/discuss/delete",
        "/data/**"
)
.hasAnyAuthority(
        AUTHORITY_ADMIN