python使用函数可以减少内存吗_Python使用DataFrame时减少内存使用的一个函数，亲测效果明显...

importpandasaspdimportnumpyasnp# @liscense: Apache 2.0

# @author: weijiandefreduce_mem_usage(props):# 计算当前内存start_mem_usg = props.memory_usage().sum() /1024**2print("Memory usage of the dataframe is :", start_mem_usg,"MB")# 哪些列包含空值，空值用-999填充。why：因为np.nan当做float处理NAlist = []forcolinprops.columns:# 这里只过滤了objectd格式，如果你的代码中还包含其他类型，请一并过滤if(props[col].dtypes !=object):print("**************************")print("columns: ", col)print("dtype before", props[col].dtype)# 判断是否是int类型isInt =Falsemmax = props[col].max()

mmin = props[col].min()# Integer does not support NA, therefore Na needs to be filledifnotnp.isfinite(props[col]).all():

NAlist.append(col)

props[col].fillna(-999, inplace=True)# 用-999填充# test if column can be converted to an integerasint = props[col].fillna(0).astype(np.int64)

result = np.fabs(props[col] - asint)

result = result.sum()ifresult <0.01:# 绝对误差和小于0.01认为可以转换的，要根据task修改isInt =True# make interger / unsigned Integer datatypesifisInt:ifmmin >=0:# 最小值大于0，转换成无符号整型ifmmax <=255:

props[col] = props[col].astype(np.uint8)elifmmax <=65535:

props[col] = props[col].astype(np.uint16)elifmmax <=4294967295:

props[col] = props[col].astype(np.uint32)else:

props[col] = props[col].astype(np.uint64)else:# 转换成有符号整型ifmmin > np.iinfo(np.int8).minandmmax < np.iinfo(np.int8).max:

props[col] = props[col].astype(np.int8)elifmmin > np.iinfo(np.int16).minandmmax < np.iinfo(np.int16).max:

props[col] = props[col].astype(np.int16)elifmmin > np.iinfo(np.int32).minandmmax < np.iinfo(np.int32).max:

props[col] = props[col].astype(np.int32)elifmmin > np.iinfo(np.int64).minandmmax < np.iinfo(np.int64).max:

props[col] = props[col].astype(np.int64)else:# 注意：这里对于float都转换成float16，需要根据你的情况自己更改props[col] = props[col].astype(np.float16)print("dtype after", props[col].dtype)print("********************************")print("___MEMORY USAGE AFTER COMPLETION:___")

mem_usg = props.memory_usage().sum() /1024**2print("Memory usage is: ", mem_usg," MB")print("This is ",100* mem_usg / start_mem_usg,"% of the initial size")returnprops, NAlist