tensorflow代码学习：队列读取tfreocrds

超级简述版

import os
import tensorflow as tf

from skimage.io import imread
import numpy as np

data_path = 'uu'
tf_data_path = 'uu/tf_data/'


def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))


def create_data(train_data_path):
    tfrecords_filename = 'a.tfrecords'
    writer = tf.python_io.TFRecordWriter(os.path.join(tf_data_path, tfrecords_filename))
    img_name = os.listdir(train_data_path)
    for image in img_name: # 遍历每一张图片
        if '.bmp' in image:
            img = imread(os.path.join(train_data_path, image), as_grey=True)  # 读取数据
            image_raw = img.tostring()
            example = tf.train.Example(features=tf.train.Features(feature={
                'image_raw': _bytes_feature(image_raw)}))
            writer.write(example.SerializeToString())
    writer.close()


def read_data(filename_queue):
    reader = tf.TFRecordReader()
    _, serialized_example = reader.read(filename_queue)
    features = tf.parse_single_example(
        serialized_example,
        # Defaults are not specified since both keys are required.
        features={
            'image_raw': tf.FixedLenFeature([], tf.string)
        })
    image = tf.decode_raw(features['image_raw'], tf.uint8)

    return image


def main():
    tfrecords_filename = ['uu/tf_data/a.tfrecords']
    filename_queue = tf.train.string_input_producer(tfrecords_filename)
    img = read_data(filename_queue)

    init_op = tf.global_variables_initializer()

    with tf.Session() as sess:
        sess.run(init_op)
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)

        image = sess.run(img)
        image = image.astype(np.uint8)
        print(image.shape)
        coord.request_stop()
        coord.join(threads)


if __name__ == '__main__':
    # create_data(data_path)
    main()

(6102656,)

我这个图片是1112*686=762832，但是输出的却是762832 * 8 = 6102656，也就是说是输出是期望的8倍，这是因为使用imread读取数据默认是float64的，也就是8个字节，
而我们最后输出的图片要求是一个字节的，这就相差了8倍
要不然，如果直接reshape(1112,686)会报错

tensorflow.python.framework.errors_impl.InvalidArgumentError: Input to reshape is a tensor with 6102656 values, but the requested shape has 762832

因为

as_grey : bool
If True, convert color images to grey-scale (64-bit floats). Images that are already in grey-scale format are not converted.

我也不知道如何转换，所以就变成cv2了，opencv默认读取的时候是直接读取三个通道的，哪怕是灰度图，而且数值类型为uint8。

---

简单版

import os
import tensorflow as tf
import cv2

data_path = 'dataset/uu/'
tf_data_path = 'dataset/uu/tf_data/'


def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))


def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))


def create_data(train_data_path):
    img_name = [img for img in os.listdir(train_data_path) if '.bmp' in img]
    for i in range(5):
        tfrecords_filename = str(i) + '.tfrecords'
        writer = tf.python_io.TFRecordWriter(os.path.join(tf_data_path, tfrecords_filename))
        for j in range(10):
            for image in img_name:
                img = cv2.imread(os.path.join(train_data_path, image))
                height, width = img.shape[0], img.shape[1]
                image_raw = img.tobytes()
                example = tf.train.Example(features=tf.train.Features(feature={
                    'height': _int64_feature(height),
                    'width': _int64_feature(width),
                    'image_raw': _bytes_feature(image_raw)
                }))
                writer.write(example.SerializeToString())
        writer.close()


def read_data(data_path):
    reader = tf.TFRecordReader()
    files = tf.train.match_filenames_once(os.path.join(data_path, '*.tfrecords'))
    #读取数据的时候忘了添加路径，只读了文件名，一直报错，请求10个数据，但是只有0个数据，让我检查了好久
    filename_queue = tf.train.string_input_producer(files, shuffle=False)
    _, serialized_example = reader.read(filename_queue)
    features = tf.parse_single_example(
        serialized_example,
        # Defaults are not specified since both keys are required.
        features={
            'height': tf.FixedLenFeature([], tf.int64),
            'width': tf.FixedLenFeature([], tf.int64),
            'image_raw': tf.FixedLenFeature([], tf.string)
        })
    height = tf.cast(features['height'], tf.int32)
    width = tf.cast(features['width'], tf.int32)
    image = tf.decode_raw(features['image_raw'], tf.uint8)
    image = tf.reshape(image, (height, width, -1))
    return image


def main():

    img = read_data(tf_data_path)

    init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())

    with tf.Session() as sess:
        sess.run(init_op)
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)

        image = sess.run(img)
        cv2.imshow('img', image)

        cv2.waitKey()
        cv2.destroyAllWindows()

        print(image)
        coord.request_stop()
        coord.join(threads)


if __name__ == '__main__':
    create_data(data_path)
    main()

这个会将所有的数据都返回来，我们训练的时候其实是想要batch训练的，因此需要改一下，例如加上

    batch_size = 10
    min_after_dequeue = 10
    capacity = min_after_dequeue + 3 * batch_size

    img_batch = tf.train.shuffle_batch([image], batch_size=batch_size, capacity=capacity,
                                       min_after_dequeue=min_after_dequeue)

但是就会报错

ValueError: All shapes must be fully defined: [TensorShape([Dimension(None), Dimension(None), Dimension(None)])]

这是因为shuffle_batch的输入必须都是尺寸已知的，，然而我们的输入image并没有明确的定义尺寸，如果你的图片大小是已知的，你可以直接在reshape的时候

image = tf.reshape(image, (420, 580, 3))

但通常不是，因此我们需要resize

image = tf.reshape(image, (height, width, 3))#先还原图片
image = tf.image.resize_images(image, (420, 580))#接着缩放

但是你会发现画出的图还是很奇怪，这是因为tf.image.resize_images函数返回的是float类型，我们将其转换成uint8就可以了

image = tf.reshape(image, (height, width, 3))#先还原图片
image = tf.image.resize_images(image, (420, 580))#接着缩放
image = tf.cast(image, tf.uint8)#转成可视化的类型，如果下面还需要进行float类型的处理，可以不用转化，这里只是为了显示

复杂版

import os
import tensorflow as tf
import cv2

data_path = 'dataset/uu/'
tf_data_path = 'dataset/uu/tf_data/'


def _bytes_feature(value):
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))


def _int64_feature(value):
    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))


def create_data(train_data_path):
    img_name = [img for img in os.listdir(train_data_path) if '.bmp' in img]
    for i in range(5):
        tfrecords_filename = str(i) + '.tfrecords'
        writer = tf.python_io.TFRecordWriter(os.path.join(tf_data_path, tfrecords_filename))
        for j in range(10):
            for image in img_name:
                img = cv2.imread(os.path.join(train_data_path, image))
                height, width = img.shape[0], img.shape[1]
                image_raw = img.tobytes()
                example = tf.train.Example(features=tf.train.Features(feature={
                    'height': _int64_feature(height),
                    'width': _int64_feature(width),
                    'image_raw': _bytes_feature(image_raw)
                }))
                writer.write(example.SerializeToString())
        writer.close()


def read_data(data_path):
    reader = tf.TFRecordReader()
    files = tf.train.match_filenames_once(os.path.join(data_path, '*.tfrecords'))
    filename_queue = tf.train.string_input_producer(files, shuffle=False)
    _, serialized_example = reader.read(filename_queue)
    features = tf.parse_single_example(
        serialized_example,
        # Defaults are not specified since both keys are required.
        features={
            'height': tf.FixedLenFeature([], tf.int64),
            'width': tf.FixedLenFeature([], tf.int64),
            'image_raw': tf.FixedLenFeature([], tf.string)
        })
    height = tf.cast(features['height'], tf.int32)
    width = tf.cast(features['width'], tf.int32)
    image = tf.decode_raw(features['image_raw'], tf.uint8)
    image = tf.reshape(image, (height, width, 3))

    image = tf.image.resize_images(image, (420, 580))
    image = tf.cast(image, tf.uint8)
    batch_size = 10
    min_after_dequeue = 10
    capacity = min_after_dequeue + 3 * batch_size

    img_batch = tf.train.shuffle_batch([image], batch_size=batch_size, capacity=capacity,
                                       min_after_dequeue=min_after_dequeue)
    return img_batch


def main():
    img = read_data(tf_data_path)

    init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())

    with tf.Session() as sess:
        sess.run(init_op)
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)

        image = sess.run(img)
        cv2.imshow('img', image[0])

        cv2.waitKey()
        cv2.destroyAllWindows()

        print(image)
        coord.request_stop()
        coord.join(threads)


if __name__ == '__main__':
    create_data(data_path)
    main()

但是有一个问题，直接使用PIL或者CV2读取像素进行存储文件会非常大，标准的是使用 tf.gfile.FastGFile进行读取

# -*- coding: utf-8 -*-
import tensorflow as tf
from PIL import Image
import numpy as np

jpeg_data = tf.placeholder(tf.string)
jpeg_decoder = tf.image.decode_jpeg(jpeg_data)

sess = tf.Session()


filename = '1.jpg'
with tf.gfile.FastGFile(filename,'rb') as f:
    image_buffer = f.read()

image = sess.run(jpeg_decoder,feed_dict={jpeg_data:image_buffer})
h,w,c = image.shape
example = tf.train.Example(features=tf.train.Features(feature={
       'h': tf.train.Feature(int64_list=tf.train.Int64List(value=[h])),
       'w': tf.train.Feature(int64_list=tf.train.Int64List(value=[w])),
       'c': tf.train.Feature(int64_list=tf.train.Int64List(value=[c])),
       'img': tf.train.Feature(bytes_list=tf.train.BytesList(value=[image_buffer]))
       }))
writer= tf.python_io.TFRecordWriter("pic.tfrecords")
writer.write(example.SerializeToString())
writer.close()



filename_queue = tf.train.string_input_producer(["pic.tfrecords"])

reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue)   #返回文件名和文件
features = tf.parse_single_example(serialized_example,features={
        'h': tf.FixedLenFeature([], tf.int64),
        'w': tf.FixedLenFeature([], tf.int64),
        'c': tf.FixedLenFeature([], tf.int64),
        'img' : tf.FixedLenFeature([], tf.string)})

img = tf.image.decode_jpeg(features['img'])

h = tf.cast(features['h'], tf.int32)
w = tf.cast(features['w'], tf.int32)
c = tf.cast(features['c'], tf.int32)

init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
with tf.Session() as sess:
    sess.run(init_op)
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(coord=coord)

    print(sess.run(img))

    coord.request_stop()
    coord.join(threads)

下面是文件大小的测试

filename = '1.jpg'
with tf.gfile.FastGFile(filename,'rb') as f:
    image_buffer = f.read()
pic = Image.open(filename)
dat = np.asarray(pic)
buf = dat.tobytes()
print('image shape {}, image size {}, image_buff size {} '.format(dat.shape,len(buf),len(image_buffer)))

输出

image shape (600, 600, 3), image size 1080000, image_buff size 34039

我也是无语了