在Tensorflow中通过以下3中方式进行读取数据:1.预加载数据(preloaded data);2.填充数据(feeding);3.从文件读取数据(reading from file);
1.预加载数据:通常通过定义常量或变量来保存所有数据,缺点:由于直接将数据嵌入数据流图中,当数据量过大时,过于消耗内存;
import tensorflow as tf
x1 = tf.constant([2,3,4])
x2 =tf.constant([4,2,1])
y = tf.add(x1,x2)
2.填充数据:由python产生数据,再把数据填充后端; 缺点:数据量大,消耗内存;以及数据类型转换过于消耗内存;
import tensorflow as tf
a1 = tf.placeholder(tf.int16)
a2 = tf.placeholder(tf.int16)
b = tf.add(a1,a2)
li1 = [2,3,4]
li2 = [4,0,1]
with tf.Session() as sess:
print(sess.run(b,feed_dict={a1:li1,a2:li2}))
3.从文件读取数据:(1)首先将样本数据写入TFReords二进制文件;(2)再从队列中读取;
实现第一步:
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import argparse
import os
import tensorflow as tf
from tensorflow.contrib.learn.python.learn.datasets import mnist
FLAGS = None
def _int64_feature(value): #生成整数类型
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def _bytes_feature(value): #生成字符类型
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def convert_to(data_set, name):
images = data_set.images
labels = data_set.labels
num_examples = data_set.num_examples
if images.shape[0] != num_examples:
raise ValueError('Images size %d does not match label size %d.' %
(images.shape[0], num_examples))
rows = images.shape[1]
cols = images.shape[2]
depth = images.shape[3]
filename = os.path.join(FLAGS.directory, name + '.tfrecords')
print('Writing', filename)
writer = tf.python_io.TFRecordWriter(filename)
for index in range(num_examples):
image_raw = images[index].tostring() #将图像矩阵转化为一个字符串
example = tf.train.Example(features=tf.train.Features(feature={
'height': _int64_feature(rows), #写入协议缓冲区,height,width,depth,label 编码成int64类型,image——raw编码成二进制
'width': _int64_feature(cols),
'depth': _int64_feature(depth),
'label': _int64_feature(int(labels[index])),
'image_raw': _bytes_feature(image_raw)}))
writer.write(example.SerializeToString())
writer.close()
def main(argv):
data_sets = mnist.read_data_sets(FLAGS.directory, #获取数据
dtype=tf.uint8,
reshape=False,
validation_size=FLAGS.validation_size)
convert_to(data_sets.train, 'train') #将数据转换成tf.train.Example类型
convert_to(data_sets.validation, 'validation')
convert_to(data_sets.test, 'test')
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument(
'--directory',
type=str,
default='D:/tmp/data',
)
parser.add_argument(
'--validation_size',
type=int,
default=5000,
)
FLAGS = parser.parse_args()
tf.app.run()
从文件中读取并解析一个样本:
def read_and_decode(filename_queue):
reader = tf.TFRecordReader()
_,serialized_example =reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'image_raw':tf.FixedLenFeature([],tf.string),
'label':tf.FixedLenFeature([],tf.int64),
})
image = tf.decode_raw(features['image_raw'],tf.uint8)
image.set_shape([mnist.IMAGE_PIXELS])
image = tf.cast(image,tf.float32)*(1./255)-0.5
label = tf.cast(features['label'],tf.int32)
return image,label