10亿条数据去重后排序和在线日志人数统计

一：10亿条数据排序

思路:数据量比较大，普通比较会占用很多的内存,可以采用其他方法，构造一个字节数组

 每个字节的值代表连续八个整形数据的值是否存在，即使包括最大的整数值，大概内存512m

源码如下

import java.util.Random;

public class BigNumberSort {


    private static final int CAPACITY = 1000000000;

    public static void main(String[] args){

        long startTime = System.currentTimeMillis();
        getRankNumber();
        long endTime = System.currentTimeMillis();

        System.out.println("\n数据排序总耗时："+(endTime-startTime));

    }

    static void getRankNumber(){

         byte[] dataByte = new byte[1<<16];

//         System.out.print("需排序的数:");
         for(int i=0;i<CAPACITY;i++){
             Random random = new Random();

             int k = random.nextInt(99);

//             System.out.print(k+"\t");

             k = k+(1<<15);

             setBit(k,dataByte);

         }

         outPutBytes(dataByte);
    }


    static void setBit(int number,byte[] dataByte){


         int byteIndex = number/8;

         if(dataByte[byteIndex]==0){
             dataByte[byteIndex] = 0;
         }

         int innerIndex = number%8;

         dataByte[byteIndex] = (byte)(dataByte[byteIndex]|(1<<innerIndex));

    }


    static void outPutBytes(byte[] dataBytes){

        int count = 0;
        System.out.println();
        for(int i=0;i<dataBytes.length;i++){

            for(int j=0;j<8;j++){

                if((dataBytes[i]&(1<<j))!=0){
                    count++;

                    int number = i*8+j-(1<<15);
                    System.out.print("第"+count+"个数: "+number+"\t");
                }
            }
        }

    }

 

 

二：统计登陆日志每秒在线人数

构造一个24*3600大小的整形数组change，根据上线时间和下线时间进行+1或者减1，统计每秒的变化数

初始大小为0,则每秒的在线人数则为online[i]=online[i-1]+change[i]

源码如下：

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;

public class OnlineNumber {


    private static int[] onlineNumber = new int[24*3600];

    private static int[] change = new int[24*3600];

    private static String filePath = "/测试";

    //获取第几秒的在线人数
    private static int i = 3662;

    //获取用户相对于零点的时间戳
    public static int getTime(String logInfo) throws ParseException {
        String s1 = logInfo.substring(5,15);
        String s2 = logInfo.substring(5,24);

        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
        Date date1 = sdf.parse(s1);
        Calendar cl = Calendar.getInstance();

        cl.setTime(date1);
        long l1 = cl.getTimeInMillis()/1000;


        SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        Date date2 = sdf2.parse(s2);
        cl.setTime(date2);
        long l2 = cl.getTimeInMillis()/1000;



        return (int) (l2 - l1);
    }

    //假设用户上线/下线时间格式:(上线/下线时间:yyyy-MM-dd HH:mm:ss)
    public static String getOnlineOrOff(String logInfo){

        String[] str = logInfo.split(":");

        return str[0];
    }

    //构造每秒在线人数的变化
    public static void fun(String filePath) throws IOException, ParseException {

        String br = "";

        FileReader fileReader = new FileReader(filePath);

        BufferedReader bufferedReader = new BufferedReader(fileReader);


        while((br=bufferedReader.readLine())!=null){

            int timestamp = getTime(br);
            String str = getOnlineOrOff(br);

            if("上线时间".equals(str)){
                change[timestamp]++;
            }else if("下线时间".equals(str)){
                change[timestamp]--;
            }
        }

        bufferedReader.close();
    }

    //获取每秒在线人数
    public static int getOnlineNumber(int i){

        if(i==0) return onlineNumber[0]+change[0];
        return getOnlineNumber(i-1)+change[i];

    }
   public static void main(String[] args) throws IOException, ParseException {

        fun(filePath);

        onlineNumber[i] = getOnlineNumber(i);

        System.out.println("第"+i+"秒的在线人数:"+onlineNumber[i]);

    }
}