Python h5py包详解

一、概念

h5py文件是存放两类对象的容器，数据集(dataset)和组(group)，dataset类似数组类的数据集合，和numpy的数组差不多。group是像文件夹一样的容器，它好比python中的字典，有键(key)和值(value)。group中可以存放dataset或者其他的group。”键”就是组成员的名称，“值”就是组成员对象本身(组或者数据集)。

总结：
一个h5py文件是 “dataset” 和 “group” 二合一的容器。

• dataset : 类似数组组织的数据的集合，像 numpy 数组一样工作
• group : 包含了其它 dataset 和 其它 group ，像字典一样工作

图解:

二、原理及优点

原理： 在简单数据的读操作中，我们通常一次性把数据全部读入到内存中。读写超过内存的大数据时，有别于简单数据的读写操作，受限于内存大小，通常需要指定位置、指定区域读写操作，避免无关数据的读写。

作用：使用h5py库读写超过内存的大数据 。

三、实战

创建hdf5文件

# -*- coding: UTF-8 -*-

import h5py
import numpy as np


def save_file():
    # ===========================================================
    # Create a HDF5 file.
    f = h5py.File("h5py_example1.hdf5", "w")  # mode = {'w', 'r', 'a'}
    print(dir(f))

    # Create two groups under root '/'.
    g1 = f.create_group("bar1")
    g2 = f.create_group("bar2")

    # Create a dataset under root '/'.
    d = f.create_dataset("dset", data=np.arange(16).reshape([4, 4]))

    # Add two attributes to dataset 'dset'
    d.attrs["myAttr1"] = [100, 200]
    d.attrs["myAttr2"] = "Hello, world!"

    # Create a group and a dataset under group "bar1".
    c1 = g1.create_group("car1")
    d1 = g1.create_dataset("dset1", data=np.arange(10))

    # Create a group and a dataset under group "bar2".
    c2 = g2.create_group("car2")
    d2 = g2.create_dataset("dset2", data=np.arange(10))

    # Save and exit the file.
    f.close()

    ''' h5py_example.hdf5 file structure
    +-- '/'
    |   +--	group "bar1"
    |   |   +-- group "car1"
    |   |   |   +-- None
    |   |   |   
    |   |   +-- dataset "dset1"  
    |   |   |   +--  [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
    |   |
    |   +-- group "bar2"
    |   |   +-- group "car2"
    |   |   |   +-- None
    |   |   |
    |   |   +-- dataset "dset2"
    |   |   |   +--  [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
    |   |   
    |   +-- dataset "dset"
    |   |   |   +-- [[ 0  1  2  3]
    |   |   |   	 [ 4  5  6  7]
    |   |   |   	 [ 8  9 10 11]
    |   |   |   	 [12 13 14 15]]
    |   |   +-- attribute "myAttr1"
    |   |   +-- attribute "myAttr2"
    |   |   
    |   
    '''

读取hdf5文件

# -*- coding: UTF-8 -*-

import h5py
import numpy as np

def load_file():
    # Read HDF5 file.
    f = h5py.File("h5py_example.hdf5", "r")  # mode = {'w', 'r', 'a'}

    # Print the keys of groups and datasets under '/'.
    print("filename: {}".format(f.filename))
    print([key for key in f.keys()], "\n")

    # ===================================================
    # Read dataset 'dset' under '/'.
    d = f["dset"]

    # Print the data of 'dset'.
    print(d.name, ":")
    print(d[:])

    # Print the attributes of dataset 'dset'.
    for key in d.attrs.keys():
        print(key, ":", d.attrs[key])

    print()

    # ===================================================
    # Read group 'bar1'.
    g = f["bar1"]

    # Print the keys of groups and datasets under group 'bar1'.
    print([key for key in g.keys()])

    # Three methods to print the data of 'dset1'.
    print(f["/bar1/dset1"][:])  # 1. absolute path

    print(f["bar1"]["dset1"][:])  # 2. relative path: file[][]

    print(g['dset1'][:])  # 3. relative path: group[]

    # Delete a database.
    """Notice: the mode should be 'a' when you read a file."""
    # del g["dset1"]

    print([key for key in g.keys()])

    # Save and exit the file
    f.close()


if __name__ == "__main__":
    save_file()

参考

https://github.com/NoNo721/Python-xamples/blob/master/HDF5/h5py_example.py

https://blog.csdn.net/qq_34859482/article/details/80115237

https://zhuanlan.zhihu.com/p/150126263

https://docs.h5py.org/en/latest/high/group.html