python【 joblib pickle cPickle savetxt】写入文件性能比较

Python2.7 测试 

import pickle
from contextlib import contextmanager
import time
import numpy as np
import joblib
import cPickle


@contextmanager
def timer(title):
    t0 = time.time()
    yield
    print("{} - done in {:.3f}s".format(title, time.time() - t0))


n=10000
VALID_NUM=1000
result = np.random.rand(n, VALID_NUM)

with timer("savetxt"):
    np.savetxt('dd.txt', result)

with timer("joblib"):
    joblib.dump(result, 'dd.txt')

with timer("pickle"):
    pickle.dump(result, open('dd.txt', 'wb'))

with timer("cPickle"):
    cPickle.dump(result, open('dd.txt', 'wb'))

结果如下：

savetxt - done in 16.336s
joblib - done in 0.349s
pickle - done in 8.560s
cPickle - done in 9.518s

 

Python3.6 测试

import pickle
from contextlib import contextmanager
import time
import numpy as np
import joblib


@contextmanager
def timer(title):
    t0 = time.time()
    yield
    print("{} - done in {:.3f}s".format(title, time.time() - t0))


n=10000
VALID_NUM=1000
result = np.random.rand(n, VALID_NUM)

with timer("savetxt"):
    np.savetxt('dd.txt', result)

with timer("joblib"):
    joblib.dump(result, 'dd.txt')

with timer("pickle"):
    pickle.dump(result, open('dd.txt', 'wb'))

结果如下：

savetxt - done in 21.417s
joblib - done in 0.282s
pickle - done in 0.454s