目录
说明
本文为博主原创文章,未经博主允许不得转载。
用netCDF4库来写nc文件的时候,如果maskandscale设置为True(默认为True),且数据变量包含scale_factor或者add_offset,读写数据时候自动进行打包和解包。
1)读取数据时候,自动解包:
data = self.scale_factor*data + self.add_offset
2)写入数据时自动打包:
data = (data - self.add_offset )/self.scale_factor
通过下面代码可以更加清楚理解这一概念。
代码实现
test_nc_write.py
# --*-- coding:utf-8 --*--
import xarray as xr
import netCDF4
import numpy as np
def calculate_scale_and_offset(minimum, maximum, n):
scale = (maximum - minimum) / (2 ** n - 1)
offset = minimum + 2 ** (n - 1) * scale
return scale, offset
# 不压缩数据
def write_nc_1(filename,data,format='NETCDF4'):
dset = netCDF4.Dataset(filename, mode='w', format=format)
# create dimensions
dset.createDimension('z', data.shape[0])
dset.createDimension('y', data.shape[1])
dset.createDimension('x', data.shape[2])
dtype = data.dtype
# use scale_factor and add_offset to pack data.
minimum = np.min(data)
maximum = np.max(data)
# dtype = 'i4'
# scale, offset = calculate_scale_and_offset(minimum,maximum, 16)
# create the dataset variable
kwargs = {'zlib': True}
ncvar = dset.createVariable('v', dtype, ('z','y','x'), **kwargs)
# ncvar.setncattr('scale_factor',scale)
# ncvar.setncattr('add_offset',offset)
ncvar[:] = data[:]
dset.close()
# 采用set_auto_maskandscale自动压缩
def write_nc_2(filename,data,format='NETCDF4'):
dset = netCDF4.Dataset(filename, mode='w', format=format)
# create dimensions
dset.createDimension('z', data.shape[0])
dset.createDimension('y', data.shape[1])
dset.createDimension('x', data.shape[2])
# use scale_factor and add_offset to pack data.
minimum = np.min(data)
maximum = np.max(data)
dtype = 'i4'
scale, offset = calculate_scale_and_offset(minimum,maximum, 16)
# create the dataset variable
kwargs = {'zlib': True}
ncvar = dset.createVariable('v', dtype, ('z','y','x'), **kwargs)
ncvar.setncattr('scale_factor',scale)
ncvar.setncattr('add_offset',offset)
ncvar[:] = data[:]
dset.close()
# 自己压缩(即不使用系统保留的scale_factor和add_offset)
def write_nc_3(filename,data,format='NETCDF4'):
dset = netCDF4.Dataset(filename, mode='w', format=format)
# create dimensions
dset.createDimension('z', data.shape[0])
dset.createDimension('y', data.shape[1])
dset.createDimension('x', data.shape[2])
# use scale_factor and add_offset to pack data.
minimum = np.min(data)
maximum = np.max(data)
dtype = 'i4'
scale, offset = calculate_scale_and_offset(minimum,maximum, 16)
# create the dataset variable
kwargs = {'zlib': True}
ncvar = dset.createVariable('v', dtype, ('z','y','x'), **kwargs)
ncvar.setncattr('v_scale',scale)
ncvar.setncattr('v_offset',offset)
# 自己对数据进行压缩
data = np.floor((data - offset) / scale)
ncvar[:] = data[:]
dset.close()
if __name__ == '__main__':
# create data to write
nx,ny,nz = 500,500,10
np.random.seed(1234)
data = np.random.rand(nz,ny,nx)
print('\n','orignal=',data[:,0,0])
file_1 = 'test_1.nc'
write_nc_1(file_1,data)
file_2 = 'test_2.nc'
write_nc_2(file_2,data)
file_3 = 'test_3.nc'
write_nc_3(file_3,data)
ds1 = xr.open_dataset(file_1)
print('\n','ds1=',ds1)
print('\n','ds1=',ds1.v.values[:,0,0])
# 默认set_auto_maskandscale=True,属性中包含scale_factor or add_offset时,
# 写入的时候,自动打包;读取的时候,自动解包,即内部完成了write_nc_3的操作
ds2 = xr.open_dataset(file_2)
print('\n','ds2=',ds2)
print('\n','ds2=',ds2.v.values[:,0,0])
# 使用的时候对数据解包
ds3 = xr.open_dataset(file_3)
offset = ds3['v'].attrs['v_offset']
scale = ds3['v'].attrs['v_scale']
print('\n','ds3=',ds3)
print('\n','ds3=',ds3.v.values[:,0,0]*scale + offset)
运行结果
采用scale_factor 和 add_offset 对浮点型数据进行压缩,能有效减少所需要的存储空间,但同时也会损失一些数据精度(如上图所示) 。
参考
1、官方文档 netCDF4 API documentation