又是一片水文
首先就是读取文件名称了标签了,我这里就是简单的猫狗识别数据集
def _find_image_files(data_dir, labels_file):
jpeg_file_path = '%s/*.jpg' % (data_dir) #文件路径
matching_files = tf.gfile.Glob(jpeg_file_path)#搜寻这个路径下的所有指定格式文件,这里是*.jpg,即所有的jpg文件
labels = [0 if 'cat' in os.path.basename(file) else 1 for file in matching_files] #根据标签的格式截取cat.0.jpg...,所以我们判断图片名称是否包含猫或狗
c = list(zip(matching_files, labels)) #文件名和对应的标签结合
shuffle(c)#将其打乱
filenames, labels = zip(*c)#解压出来后就是打乱后的数据了
return filenames, labels
```python
接着指定一下需要保存的数据格式,就是example,这个是tensorflow指定的数据结构,我这里就是只保存图片的数据和标签
```python
def _convert_to_example(image_buffer, label):
example = tf.train.Example(features=tf.train.Features(feature={
'image/label': _int64_feature(label),
'image/encoded': _bytes_feature(image_buffer)}))
return example
然后就是把写数据了,我们可以将所有的图片和标签写到一个文件,也可以分开写到几个文件中,首先就是把图片和标签分割成几部分,然后这几段数据分别写到不同的tfrecords中。
def _process_image_files(output_directory, name, filenames, labels, num_shards):
num_images = len(filenames) # 看看有多少张图片
num_batch = np.linspace(0,num_images,num_shards+1).astype(np.int) # 将数据划分为n块,可以看一下np.linspace就知道为什么要加1了
for counter in range(num_shards):
output_filename = '%s-%.5d-of-%.5d' % (name, counter, num_shards) #要保存的tfrecords文件名,train-00001-00008(名字,写入的第几个文件,总共几个文件)
output_file = os.path.join(output_directory, output_filename)
start,end = num_batch[counter], num_batch[counter+1]#获得每一段数据的起始范围
writer = tf.python_io.TFRecordWriter(output_file)#写文件描述符
for j in range(start,end): #遍历这段区间的所有文件
filename,label = filenames[j],labels[j]
try:
with tf.gfile.FastGFile(filename,'rb') as f:
image_buffer = f.read() #读取图片的原始数据
except Exception as e:
print(e)
continue
example = _convert_to_example(image_buffer, label)#将数据和标签保存成指定的example格式
writer.write(example.SerializeToString())
print('writing {} picture, filename is {}, label is {}, shard is {}'.format(j,filename,label,counter))
writer.close()
整合一下
def _process_dataset(output_directory,name, directory, labels_file, num_shards):
filenames, labels = _find_image_files(directory, labels_file)
_process_image_files(output_directory,name, filenames, labels,num_shards)
def main(unused_argv):
_process_dataset('tfdata','train', '../../cat_dog/train', '', 8)
# 猫狗数据没有具体的标签文件,所以直接为空,如果有的话自己处理
if __name__ == '__main__':
tf.app.run()
执行过程的输出
writing 24993 picture, filename is ..\..\cat_dog\train\dog.10861.jpg, label is 1, shard is 7
writing 24994 picture, filename is ..\..\cat_dog\train\dog.7031.jpg, label is 1, shard is 7
writing 24995 picture, filename is ..\..\cat_dog\train\cat.7885.jpg, label is 0, shard is 7
writing 24996 picture, filename is ..\..\cat_dog\train\dog.8770.jpg, label is 1, shard is 7
writing 24997 picture, filename is ..\..\cat_dog\train\dog.6193.jpg, label is 1, shard is 7
writing 24998 picture, filename is ..\..\cat_dog\train\cat.11390.jpg, label is 0, shard is 7
writing 24999 picture, filename is ..\..\cat_dog\train\cat.5946.jpg, label is 0, shard is 7
文件写完之后我们就可以读取了,首先就是把写好的tfrecords文件加载进来
dataset = tf.data.TFRecordDataset(filenames)
有了这个数据集我们就可以为所欲为了,比如我们想要打乱数据可以使用dataset.buffle(1024)
,我们想要重复几次,其实就是循环训练几次数据集,可以使用dataset.repeat()
,等等等。。。,当然干这些之前首先要把我们的数据重example的格式解析成图片格式,需要dataset.map(parse_function)
这个函数
def _parse_function(example_proto):
features = {
'image/label': tf.FixedLenFeature((), tf.int64, default_value=0),
'image/encoded': tf.FixedLenFeature((), tf.string, default_value="")
}
parsed = tf.parse_single_example(example_proto, features) #解析example
label = tf.cast(parsed['image/label'], tf.int32)
encoded = tf.image.decode_image(parsed['image/encoded'])
#encoded = tf.image.decode_jpeg(parsed['image/encoded']) #这个就不需要加下面的那句了
encoded.set_shape([None, None, None])#一定要加这一句,不信你试试,
encoded = tf.image.resize_images(encoded,(224,224)) #如果不缩放到相同的尺寸,就无法batch读取,这个真不是很好,还不如自己读取batch,例如原始的多线程队列读取
return encoded, label
下面将所有的代码汇总一下把
# -*- coding: utf-8 -*-
# -*- coding: utf-8 -*-
import tensorflow as tf
import six
import os
import numpy as np
from random import shuffle
def _int64_feature(value):
if not isinstance(value, list):
value = [value]
return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
def _float_feature(value):
"""Wrapper for inserting float features into Example proto."""
if not isinstance(value, list):
value = [value]
return tf.train.Feature(float_list=tf.train.FloatList(value=value))
def _bytes_feature(value):
"""Wrapper for inserting bytes features into Example proto."""
if isinstance(value, six.string_types):
value = six.binary_type(value, encoding='utf-8')
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def _convert_to_example(image_buffer, label):
example = tf.train.Example(features=tf.train.Features(feature={
'image/label': _int64_feature(label),
'image/encoded': _bytes_feature(image_buffer)}))
return example
def _process_image_files(output_directory, name, filenames, labels, num_shards):
num_images = len(filenames)
num_batch = np.linspace(0,num_images,num_shards+1).astype(np.int) # 将数据划分为n块[0,100,200...]
for counter in range(num_shards):
output_filename = '%s-%.5d-of-%.5d' % (name, counter, num_shards)
output_file = os.path.join(output_directory, output_filename)
start,end = num_batch[counter], num_batch[counter+1]
writer = tf.python_io.TFRecordWriter(output_file)
for j in range(start,end):
filename,label = filenames[j],labels[j]
try:
with tf.gfile.FastGFile(filename,'rb') as f:
image_buffer = f.read()
except Exception as e:
print(e)
continue
example = _convert_to_example(image_buffer, label)
writer.write(example.SerializeToString())
print('writing {} picture, filename is {}, label is {}, shard is {}'.format(j,filename,label,counter))
writer.close()
def _find_image_files(data_dir, labels_file):
jpeg_file_path = '%s/*.jpg' % (data_dir)
matching_files = tf.gfile.Glob(jpeg_file_path)
labels = [0 if 'cat' in os.path.basename(file) else 1 for file in matching_files]
c = list(zip(matching_files, labels))
shuffle(c)
filenames, labels = zip(*c)
return filenames, labels
def _process_dataset(output_directory,name, directory, labels_file, num_shards):
filenames, labels = _find_image_files(directory, labels_file)
_process_image_files(output_directory,name, filenames, labels,num_shards)
def main(unused_argv):
_process_dataset('tfdata','train', '../../cat_dog/train', '', 8)
def input_function(filenames):
def _parse_function(example_proto):
features = {
'image/label': tf.FixedLenFeature((), tf.int64, default_value=0),
'image/encoded': tf.FixedLenFeature((), tf.string, default_value="")
}
parsed = tf.parse_single_example(example_proto, features)
label = tf.cast(parsed['image/label'], tf.int32)
encoded = tf.image.decode_image(parsed['image/encoded'])
encoded.set_shape([None, None, None])
encoded = tf.image.resize_images(encoded,(224,224)) #如果不缩放到相同的尺寸,就无法读取
return encoded, label
dataset = tf.data.TFRecordDataset(filenames)
dataset = dataset.map(_parse_function)
dataset = dataset.repeat()
dataset = dataset.batch(32)
iterator = dataset.make_one_shot_iterator()
next_element = iterator.get_next()
return next_element
if __name__ == '__main__':
tf.app.run()
我们可以验证读取数据
# -*- coding: utf-8 -*-
import tensorflow as tf
from build1 import input_function
import matplotlib.pyplot as plt
import numpy as np
filenames = ['tfdata/train-00000-of-00008','tfdata/train-00001-of-00008']
next_element = input_function(filenames)
with tf.Session() as sess:
img,lab = sess.run(next_element)
plt.imshow(img[0].astype(np.int))
print(img[0].shape)
while True:
try:
print(sess.run(next_element))
except tf.errors.OutOfRangeError:
break