Tensorflow生成自己的图片数据集TFrecords

Tensorflow生成自己的图片数据集TFrecords

train.txt保存图片的路径和标签信息

1. test_image/dog/ 1.jpg 0
2. test_image/dog/ 2.jpg 0
3. test_image/dog/ 3.jpg 0
4. test_image/dog/ 4.jpg 0
5. test_image/cat/ 1.jpg 1
6. test_image/cat/ 2.jpg 1
7. test_image/cat/ 3.jpg 1
8. test_image/cat/ 4.jpg 1

使用下面完整代码，可以生成自己的图片数据集TFrecords，并解析：

1. # -*- coding: utf-8 -*-
2. # !/usr/bin/python3.5
3. # ref url : https://blog.csdn.net/guyuealian/article/details/80857228
4. # Author : pan_jinquan
5. # Date : 2018.6.29
6. # Function: image convert to tfrecords
7. #############################################################################################
9. import tensorflow as tf
10. import numpy as np
11. import matplotlib.pyplot as plt
12. from PIL import Image
14. # 参数设置
15. ###############################################################################################
16. train_file = 'train.txt' # 图片路径
17. output_record_dir= './tfrecords/my_record.tfrecords'
18. resize_height = 100 # 指定存储图片高度
19. resize_width = 100 # 指定存储图片宽度
22. ###############################################################################################
23. # 生成整数型的属性
24. def _int64_feature(value):
25. return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
26. # 生成字符串型的属性
27. def _bytes_feature(value):
28. return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
29. # 生成实数型的属性
30. def float_list_feature(value):
31. return tf.train.Feature(float_list=tf.train.FloatList(value=value))
33. # 显示图片
34. def show_image(image_name,image):
35. # plt.figure("show_image") # 图像窗口名称
36. plt.imshow(image)
37. plt.axis( 'on') # 关掉坐标轴为 off
38. plt.title(image_name) # 图像题目
39. plt.show()
41. #载图txt文件，文件中每行为一个图片信息，且以空格隔开：图像路径 标签，如：test_image/1.jpg 0
42. def load_txt_file(examples_list_file):
43. lines = np.genfromtxt(examples_list_file, delimiter= " ", dtype=[( 'col1', 'S120'), ( 'col2', 'i8')])
44. examples = []
45. labels = []
46. for example, label in lines:
47. examples.append(example.decode( 'ascii'))
48. labels.append(label)
49. return np.asarray(examples), np.asarray(labels), len(lines)
51. # 读取原始图像数据
52. def read_image(filename, resize_height, resize_width):
53. # image = cv2.imread(filename)
54. # image = cv2.resize(image, (resize_height, resize_width))
55. # b, g, r = cv2.split(image)
56. # rgb_image = cv2.merge([r, g, b])
57. rgb_image=Image.open(filename)
58. rgb_image=rgb_image.resize((resize_width,resize_height))
59. image=np.asanyarray(rgb_image)
60. # show_image("src resize image",image)
61. return image
64. #保存record文件
65. def create_record(train_file, output_record_dir, resize_height, resize_width):
66. _examples, _labels, examples_num = load_txt_file(train_file)
67. writer = tf.python_io.TFRecordWriter(output_record_dir)
68. for i, [example, label] in enumerate(zip(_examples, _labels)):
69. print( 'No.%d' % (i))
70. image = read_image(example, resize_height, resize_width)
71. print( 'shape: %d, %d, %d, label: %d' % (image.shape[ 0], image.shape[ 1], image.shape[ 2], label))
72. image_raw = image.tostring()
73. example = tf.train.Example(features=tf.train.Features(feature={
74. 'image_raw': _bytes_feature(image_raw),
75. 'height': _int64_feature(image.shape[ 0]),
76. 'width': _int64_feature(image.shape[ 1]),
77. 'depth': _int64_feature(image.shape[ 2]),
78. 'label': _int64_feature(label)
79. }))
80. writer.write(example.SerializeToString())
81. writer.close()
83. #解析record文件，并显示，主要用于验证
84. def disp_records(tfrecord_list_file):
85. filename_queue = tf.train.string_input_producer([tfrecord_list_file])
86. reader = tf.TFRecordReader()
87. _, serialized_example = reader.read(filename_queue)
88. features = tf.parse_single_example(
89. serialized_example,
90. features={
91. 'image_raw': tf.FixedLenFeature([], tf.string),
92. 'height': tf.FixedLenFeature([], tf.int64),
93. 'width': tf.FixedLenFeature([], tf.int64),
94. 'depth': tf.FixedLenFeature([], tf.int64),
95. 'label': tf.FixedLenFeature([], tf.int64)
96. }
97. )
98. image = tf.decode_raw(features[ 'image_raw'], tf.uint8)
99. # print(repr(image))
100. height = features[ 'height']
101. width = features[ 'width']
102. depth = features[ 'depth']
103. label = tf.cast(features[ 'label'], tf.int32)
104. init_op = tf.initialize_all_variables()
105. resultImg = []
106. resultLabel = []
107. with tf.Session() as sess:
108. sess.run(init_op)
109. coord = tf.train.Coordinator()
110. threads = tf.train.start_queue_runners(sess=sess, coord=coord)
111. for i in range( 4):
112. image_eval = image.eval()
113. resultLabel.append(label.eval())
114. image_eval_reshape = image_eval.reshape([height.eval(), width.eval(), depth.eval()])
115. print( "image.shape=",height.eval(), width.eval(), depth.eval())
116. resultImg.append(image_eval_reshape)
117. # pilimg = Image.fromarray(np.asarray(image_eval_reshape))
118. # pilimg.show()
119. show_image( "decode_from_tfrecords",image_eval_reshape)
120. coord.request_stop()
121. coord.join(threads)
122. sess.close()
123. return resultImg, resultLabel
125. #解析record文件
126. def read_record(filename_queuetemp):
127. filename_queue = tf.train.string_input_producer([filename_queuetemp])
128. reader = tf.TFRecordReader()
129. _, serialized_example = reader.read(filename_queue)
130. features = tf.parse_single_example(
131. serialized_example,
132. features={
133. 'image_raw': tf.FixedLenFeature([], tf.string),
134. 'width': tf.FixedLenFeature([], tf.int64),
135. 'depth': tf.FixedLenFeature([], tf.int64),
136. 'label': tf.FixedLenFeature([], tf.int64)
137. }
138. )
139. image = tf.decode_raw(features[ 'image_raw'], tf.uint8) #获得图像原始的数据
140. image=tf.reshape(image, [ 100, 100, 3]) # 设置图像的维度
141. # image = tf.cast(image, tf.float32) * (1. / 255) - 0.5 # 归一化
142. label = tf.cast(features[ 'label'], tf.int32) # label
143. return image, label
147. def test():
148. create_record(train_file, output_record_dir, resize_height, resize_width) # 产生record文件
150. # img, label = disp_records(output_record_dir) # 显示函数
152. image_data, label_data= read_record(output_record_dir) # 读取函数
155. #使用shuffle_batch可以随机打乱输入
156. image_batch, label_batch = tf.train.shuffle_batch([image_data, label_data], batch_size= 30, capacity= 2000,min_after_dequeue= 1000)
158. init = tf.global_variables_initializer()
159. with tf.Session() as sess: #开始一个会话
160. sess.run(init)
161. coord = tf.train.Coordinator()
162. threads = tf.train.start_queue_runners(coord=coord)
163. for i in range( 4):
164. images, labels= sess.run([image_batch, label_batch]) #在会话中取出image和label
165. #我们也可以根据需要对val， l进行处理
166. #l = to_categorical(l, 12)
167. show_image( "image",images[i,:,:,:])
168. print(images.shape, labels)
169. #停止所有线程
170. coord.request_stop()
171. coord.join(threads)
172. sess.close() #关闭会话
175. if __name__ == '__main__':
176. test()

假设我们已经生成了output.tfrecords，其中保存有：

1. image_raw：图像的特征向量，有 784维
2. pixels：图像分辨率大小 28
3. label：图像的标签

 可以使用下面的方法，进行训练网络：

1. #coding=utf-8
2. import tensorflow as tf
4. # 模型相关的参数
5. INPUT_NODE = 784
6. OUTPUT_NODE = 10
7. LAYER1_NODE = 500
8. REGULARAZTION_RATE = 0.0001
9. TRAINING_STEPS = 5000
12. files = tf.train.match_filenames_once( "./output.tfrecords")
13. filename_queue = tf.train.string_input_producer(files, shuffle= False)
16. # 读取文件。
18. reader = tf.TFRecordReader()
19. _,serialized_example = reader.read(filename_queue)
21. # 解析读取的样例。
22. features = tf.parse_single_example(
23. serialized_example,
24. features={
25. 'image_raw':tf.FixedLenFeature([],tf.string),
26. 'pixels':tf.FixedLenFeature([],tf.int64),
27. 'label':tf.FixedLenFeature([],tf.int64)
28. })
30. decoded_images = tf.decode_raw(features[ 'image_raw'],tf.uint8)
31. retyped_images = tf.cast(decoded_images, tf.float32)
32. labels = tf.cast(features[ 'label'],tf.int32)
33. #pixels = tf.cast(features['pixels'],tf.int32)
34. images = tf.reshape(retyped_images, [ 784])
35. min_after_dequeue = 10000
36. batch_size = 100
37. capacity = min_after_dequeue + 3 * batch_size
39. image_batch, label_batch = tf.train.shuffle_batch([images, labels],
40. batch_size=batch_size,
41. capacity=capacity,
42. min_after_dequeue=min_after_dequeue)
43. def inference(input_tensor, weights1, biases1, weights2, biases2):
44. layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)
45. return tf.matmul(layer1, weights2) + biases2
49. weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev= 0.1))
50. biases1 = tf.Variable(tf.constant( 0.1, shape=[LAYER1_NODE]))
52. weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev= 0.1))
53. biases2 = tf.Variable(tf.constant( 0.1, shape=[OUTPUT_NODE]))
55. y = inference(image_batch, weights1, biases1, weights2, biases2)
57. # 计算交叉熵及其平均值
58. cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=label_batch)
59. cross_entropy_mean = tf.reduce_mean(cross_entropy)
61. # 损失函数的计算
62. regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
63. regularaztion = regularizer(weights1) + regularizer(weights2)
64. loss = cross_entropy_mean + regularaztion
66. # 优化损失函数
67. train_step = tf.train.GradientDescentOptimizer( 0.01).minimize(loss)
69. # 初始化回话并开始训练过程。
70. with tf.Session() as sess:
71. tf.global_variables_initializer().run()
72. coord = tf.train.Coordinator()
73. threads = tf.train.start_queue_runners(sess=sess, coord=coord)
74. # 循环的训练神经网络。
75. for i in range(TRAINING_STEPS):
76. if i % 1000 == 0:
77. print( "After %d training step(s), loss is %g " % (i, sess.run(loss)))
79. sess.run(train_step)
80. coord.request_stop()
81. coord.join(threads)