python 性能测试（一）：多进程、向量化、numpy

import math
import numpy as np
from datetime import datetime as dt
from multiprocessing import Pool
def fun(n):
    if n % 2 == 0:
        return False
    from_i = 3
    to_i = math.sqrt(n) + 1
    for i in range(from_i,int(to_i),2):
        if n % i == 0:
            return False
    return True

#numpy数组
def NP(n):
    # t0 = dt.now()
    nums = 2748779069441 * np.ones(n)
    ret = [False]*n
    for i in range(len(nums)):
        ret[i] = fun(nums[i])
    # t1 = dt.now()
    # print((t1-t0).total_seconds())
    # print(sum(ret))
#numpy数组+函数向量化
def NPVec(n):
    # t0 = dt.now()
    nums = 2748779069441 * np.ones(n)
    fun_vecterize = np.vectorize(fun,otypes = [bool])
    ret = fun_vecterize(nums)
    # t1 = dt.now()
    # print((t1-t0).total_seconds())
    # print(sum(ret))
#numpy数组+多进程
def NPPool(n,pN):
    # t0 = dt.now()
    nums = [2748779069441] * n
    p = Pool(pN)
    ret = p.map(fun,nums)
    p.close()
    p.join()
    # t1 = dt.now()
    # print('#',pN,(t1-t0).total_seconds())
    # print(sum(ret))
#numpy数组+多进程+函数向量化
def NPVecPool(n,pN):
    # t0 = dt.now()
    nums = 2748779069441 * np.ones(n//pN)
    remain = n % pN
    numsFirst = 2748779069441 * np.ones(n//pN + remain)
    Nums = [numsFirst]
    for i in range(pN - 1):
        Nums.append(nums)
    p = Pool(pN)
    fun_vecterize = np.vectorize(fun,otypes = [bool])
    ret = p.map(fun_vecterize,Nums)
    p.close()
    p.join()
    # t1 = dt.now()
    # print(pN,(t1-t0).total_seconds())
    # print(sum(sum(ret)))    

运行环境：python3+win10 64位+ 16核 32超线程

命令行测试：

> python -m timeit -s "import hellopy" "hellopy.NP(64)"
1 loop, best of 5: 33.1 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPVec(64)"
1 loop, best of 5: 5.13 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPPool(64,8)"
1 loop, best of 5: 1.21 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPVecPool(64,8)" 
1 loop, best of 5: 1.02 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPPool(64,16)"
1 loop, best of 5: 877 msec per loop
> python -m timeit -s "import hellopy" "hellopy.NPVecPool(64,16)"
1 loop, best of 5: 811 msec per loop
> python -m timeit -s "import hellopy" "hellopy.NPPool(64,32)"
1 loop, best of 5: 1.4 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPVecPool(64,32)"
1 loop, best of 5: 1.41 sec per loop

可以看出多进程和向量化可以明显提高速度。下面增加数据量，观察向量化的速度提升效果

> python -m timeit -s "import hellopy" "hellopy.NPPool(1024,8)"
1 loop, best of 5: 14.8 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPVecPool(1024,8)"
1 loop, best of 5: 11.4 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPPool(1024,16)" 
1 loop, best of 5: 7.66 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPVecPool(1024,16)"
1 loop, best of 5: 6.41 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPPool(1024,32)"
1 loop, best of 5: 8.94 sec per loop
> python -m timeit -s "import hellopy" "hellopy.NPVecPool(1024,32)"
1 loop, best of 5: 7.71 sec per loop

结论：并行计算+向量化可以明显提高算速度