减小文件导入内存

摘要

基于类的形式

class ReduceUsage(object):
    
    def __init__(self, **kwargs):
        self.file = kwargs.get('file')
        self.encoding = kwargs.get('encoding')

    def reduce_mem_usage(self):
        """
         iterate through all the columns of
         a data and modify the data type
         to reduce memory usage.
        """
        df = self.import_data()
        import numpy as np
        start_mem = df.memory_usage().sum() / 1024 ** 2
        print('Memory usage of Data is {:.2f} MB'.format(start_mem))

        for col in df.columns:
            col_type = df[col].dtype

            if col_type != object:
                c_min = df[col].min()
                c_max = df[col].max()
                if str(col_type)[:3] == 'int':
                    if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                        df[col] = df[col].astype(np.int8)
                    elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                        df[col] = df[col].astype(np.int16)
                    elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                        df[col] = df[col].astype(np.int32)
                    elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                        df[col] = df[col].astype(np.int64)
                else:
                    if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                        df[col] = df[col].astype(np.float16)
                    elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                        df[col] = df[col].astype(np.float32)
                    else:
                        df[col] = df[col].astype(np.float64)
            else:
                df[col] = df[col].astype('category')

        end_mem = df.memory_usage().sum() / 1024 ** 2
        print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
        print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))

        return df

    def import_data(self):
        """create a data and optimize its memory usage"""
        import pandas as pd
        df_1 = pd.read_csv(self.file, parse_dates=True, keep_date_col=True, encoding=self.encoding)
        return df_1


if __name__ == '__main__':
    import os
    os.chdir('D:\Vshare')
    file = 'train_data.csv'
    encoding = 'utf-8'
    kw = {'file': file,
          'encoding': encoding}
    ru = ReduceUsage(**kw).reduce_mem_usage()

1.定义输出文件与编码格式
2.输出的结果如下