统计类优化算法初步

public class testcache {

	class A{
	 public	int []tagid;
	 public	int []tagvalueid;
	}
	public A [] tmpA;
	public  void test(){
//		  tmpA=new A[20000000];
		tmpA=new A[20000000];
		for (int i = 0; i < tmpA.length; i++) {
			tmpA[i]=new A();
			tmpA[i].tagid=new int[10];
			for (int j = 0; j < tmpA[i].tagid.length; j++) {
				tmpA[i].tagid[j]=j;
			}
			tmpA[i].tagvalueid=new int [10];

			for (int j = 0; j < tmpA[i].tagvalueid.length; j++) {
				tmpA[i].tagvalueid[j]=j;
			}
		}
	}
	/**
	 * @param args
	 */
	public static void main(String[] args) {
		// TODO Auto-generated method stub
		  System.out.println("freeMemory="+Runtime.getRuntime().freeMemory());   
		  System.out.println("totalMemory="+Runtime.getRuntime().totalMemory());   
		  System.out.println("maxMemory="+Runtime.getRuntime().maxMemory());
		  System.out.println("--------------------------------");
		long start=System.currentTimeMillis();
//		int [] tmp=new int[100000000];
//		for (int i = 0; i < tmp.length; i++) {
//			tmp[i]=i;
//		}
//		String [] tmp=new String[10000000];
//		for (int i = 0; i < tmp.length; i++) {
//			tmp[i]=""+i;
//		}

		BirdBitSet tmpBitSet1=new BirdBitSet();
		tmpBitSet1.set(1);
		tmpBitSet1.set(1000);
		tmpBitSet1.set(99999);
		System.out.println("tmpBitSet1.size()="+tmpBitSet1.size());

		testcache _testcache=new testcache();
		_testcache.test();
		System.out.println("time1:"+(System.currentTimeMillis()-start));
		BirdBitSet tmpBitSet=new BirdBitSet();

		start=System.currentTimeMillis();
		int tmpi=0;
		for (int i = 0; i < 10000000; i++) {
			tmpi=(int) Math.round(Math.random()*20000001);

			tmpBitSet.set(tmpi);
		}
		System.out.println("time2:"+(System.currentTimeMillis()-start));
		start=System.currentTimeMillis();
		int iranddom=0;

		for (int i = tmpBitSet.nextSetBit(0); i >=0; i=tmpBitSet.nextSetBit(i+1)) {

			int tmpicount=_testcache.tmpA[i].tagid.length;
			for (int j = 0; j < tmpicount; j++) {
				int tmp=_testcache.tmpA[i].tagid[j]+100;
			}
			int tmpicount1=_testcache.tmpA[i].tagvalueid.length;
			for (int j = 0; j < tmpicount; j++) {
				int tmp=_testcache.tmpA[i].tagvalueid[j]+100;
			}
		}

		System.out.println("iranddom:"+(iranddom));
		System.out.println("time3:"+(System.currentTimeMillis()-start));


		  System.out.println("freeMemory="+Runtime.getRuntime().freeMemory());   
		  System.out.println("totalMemory="+Runtime.getRuntime().totalMemory());   
		  System.out.println("maxMemory="+Runtime.getRuntime().maxMemory());
		  System.out.println("totalMemory-freeMemory[已经使用的内存]="+(Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory())/1024/1024);

	}

}

结论

单线程 8核机器

2000w
随机取1000w
freeMemory=8588320
totalMemory=9109504
maxMemory=4727504896
--------------------------------
初始数据
time1:88881
模似数据
time2:1172
iranddom:0
//取值
time3:1324
freeMemory=670798848
totalMemory=4201250816
maxMemory=4727504896
totalMemory-freeMemory[已经使用的内存]=3366 (3.3G)

如果用多线程将取到很好的效果,16个核的机器估计可以支撑1亿记录数据量的聚类