paddle复现pytorch踩坑(十）：dataloader读取

问题一：图片大小问题

报错：

F:\Anaconda\envs\paddle\lib\site-packages\paddle\fluid\reader.py:811: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
  arr = np.array(item)
2020-12-15 16:30:02,138-WARNING: DygraphDataLoader reader thread raised an exception.
Traceback (most recent call last):
  File "F:/Programs/3DOB/D4LCNpaddle/try.py", line 95, in <module>
    main()
  File "F:/Programs/3DOB/D4LCNpaddle/try.py", line 90, in main
    for idx, (data, label) in enumerate(dataloader):
  File "F:\Anaconda\envs\paddle\lib\site-packages\paddle\fluid\reader.py", line 804, in __next__
    return self._reader.read_next_var_list()
paddle.fluid.core_avx.EnforceNotMet: 

--------------------------------------------
C++ Call Stacks (More useful to developers):
--------------------------------------------
Windows not support stack backtrace yet.

----------------------
Error Message Summary:
----------------------
Error: Blocking queue is killed because the data reader raises an exception
  [Hint: Expected killed_ != true, but received killed_:1 == true:1.] at (D:\1.8.0cpu\paddle\paddle/fluid/operators/reader/blocking_queue.h:141)

解决：
利用transformer将图片size调整到统一size

问题二：paddle无法读取easydict

问题：paddlepaddle自带的dataloader无法读取字典类型（dict）
解决：采用迭代器，yeild方法写自己的dataloader

import cv2
import numpy as np

class Transform(object):
    def __init__(self, height=1242, length=375):
        self.height = height
        self.length = length

    def __call__(self, rgb, depth):
        rgb = cv2.resize(rgb, (self.height, self.length), interpolation=cv2.INTER_LINEAR)
        depth = cv2.resize(depth, (self.height, self.length), interpolation=cv2.INTER_NEAREST)

        return rgb, depth

class Reader(object):

    def __init__(self,
                 conf,
                 list_file,
                 transform=None):

        self.conf = conf
        self.list_file = list_file
        self.data_list = self.read_list()
        self.batch_size = conf.batch_size
        self.transform = transform
        self.len = len(self.data_list)

    # 读取数据目录
    def read_list(self):
        data_list = []
        with open(self.list_file) as infile:
            for line in infile:
                data_list.append(line.split()[0])
        return data_list

    # 正式加载数据的部分
    def create_reader(self):

        labelreader = self._reader_creator()

        def _batch_reader():
            batch_img = []
            batch_depth = []
            batch_label = []

            for index, imobj in enumerate(labelreader):
                if imobj is None:
                    continue

                rgb = cv2.imread(imobj.path, cv2.IMREAD_COLOR).astype(np.float32)
                depth = cv2.imread(imobj.path_depth, cv2.IMREAD_COLOR).astype(np.float32)

                # transform 里面需要做两件事
                # 1. 数据预处理
                # 2. 所有图片进行对齐 resize
                rgb, depth, label = self.transform(rgb, depth, imobj)

                rgb = np.transpose(rgb, [2, 0, 1])
                depth = np.transpose(depth, [2, 0, 1])

                batch_img.append(rgb)
                batch_depth.append(depth)
                batch_label.append(label)

                # 判断以生成指定batch的数据块
                if len(batch_img) == self.batch_size:
                    yield batch_img, batch_depth, batch_label
                    batch_img, batch_depth, batch_label = [], [], []

        return _batch_reader

    def _reader_creator(self):

        imdb = edict()

        # TODO
        # 这部分读取字典标签所需要的信息

        return imdb