TensorFlow的队列
程序10-1：
import tensorflow as tf
with tf.Session() as sess:
q = tf.FIFOQueue(3, "float")#设定一个先入先出的队列
init = q.enqueue_many(([0.1, 0.2, 0.3],))#将零个或多个元素编入队列
init2 = q.dequeue()#将元素从队列移出 0.1出列
init3 = q.enqueue(1.)#将该元素编入队列 1入列
sess.run(init)
sess.run(init2)
sess.run(init3)
quelen = sess.run(q.size())
for i in range(quelen):
print(sess.run(q.dequeue()))
结果：
0.2
0.3
1.0
QueueRunner函数解决异步操作问题
程序10-2：
import tensorflow as tf
with tf.Session() as sess:
# sess=tf.Session()
q = tf.FIFOQueue(1000, "float32")
counter = tf.Variable(0.0)
add_op = tf.assign_add(counter, tf.constant(1.0))#将整数1叠加到变量counter上去
enqueueData_op = q.enqueue(counter)
#创建一个队列管理器QueueRunner，调用2个线程去完成此项任务
qr = tf.train.QueueRunner(q, enqueue_ops=[add_op, enqueueData_op] * 2)
sess.run(tf.global_variables_initializer())
enqueue_threads = qr.create_threads(sess, start=True) # 启动入队线程
for i in range(10):
print(sess.run(q.dequeue()))
结果报错：
4.0
6.0
8.0
8.0
8.0
10.0
10.0
12.0
12.0
14.0
E0722 21:35:53.848601 9960 queue_runner_impl.py:275] Exception in QueueRunner: Run call was cancelled
E0722 21:35:53.848601 3200 queue_runner_impl.py:275] Exception in QueueRunner: Run call was cancelled
E0722 21:35:53.848601 10328 queue_runner_impl.py:275] Exception in QueueRunner: [_Derived_]Enqueue operation was cancelled
[[{{node fifo_queue_enqueue}}]]
E0722 21:35:53.848601 10592 queue_runner_impl.py:275] Exception in QueueRunner: [_Derived_]Enqueue operation was cancelled
[[{{node fifo_queue_enqueue}}]]
Exception in thread QueueRunnerThread-fifo_queue-fifo_queue_enqueue:
Traceback (most recent call last):
File "D:\ProgramData\Anaconda3\lib\threading.py", line 917, in _bootstrap_inner
self.run()
File "D:\ProgramData\Anaconda3\lib\threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\training\queue_runner_impl.py", line 257, in _run
enqueue_callable()
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1279, in _single_operation_run
self._call_tf_sessionrun(None, {}, [], target_list, None)
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1429, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.CancelledError: [_Derived_]Enqueue operation was cancelled
[[{{node fifo_queue_enqueue}}]]
Exception in thread QueueRunnerThread-fifo_queue-AssignAdd:0:
Traceback (most recent call last):
File "D:\ProgramData\Anaconda3\lib\threading.py", line 917, in _bootstrap_inner
self.run()
File "D:\ProgramData\Anaconda3\lib\threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\training\queue_runner_impl.py", line 257, in _run
enqueue_callable()
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1289, in _single_tensor_run
results = self._call_tf_sessionrun(None, {}, fetch_list, [], None)
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1429, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled
Exception in thread QueueRunnerThread-fifo_queue-fifo_queue_enqueue:
Traceback (most recent call last):
File "D:\ProgramData\Anaconda3\lib\threading.py", line 917, in _bootstrap_inner
self.run()
File "D:\ProgramData\Anaconda3\lib\threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\training\queue_runner_impl.py", line 257, in _run
enqueue_callable()
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1279, in _single_operation_run
self._call_tf_sessionrun(None, {}, [], target_list, None)
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1429, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.CancelledError: [_Derived_]Enqueue operation was cancelled
[[{{node fifo_queue_enqueue}}]]
Exception in thread QueueRunnerThread-fifo_queue-AssignAdd:0:
Traceback (most recent call last):
File "D:\ProgramData\Anaconda3\lib\threading.py", line 917, in _bootstrap_inner
self.run()
File "D:\ProgramData\Anaconda3\lib\threading.py", line 865, in run
self._target(*self._args, **self._kwargs)
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\training\queue_runner_impl.py", line 257, in _run
enqueue_callable()
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1289, in _single_tensor_run
results = self._call_tf_sessionrun(None, {}, fetch_list, [], None)
File "D:\ProgramData\Anaconda3\lib\site-packages\tensorflow\python\client\session.py", line 1429, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.CancelledError: Run call was cancelled
程序10-3：
import tensorflow as tf
q = tf.FIFOQueue(1000, "float32")
counter = tf.Variable(0.0)
add_op = tf.assign_add(counter, tf.constant(1.0))
enqueueData_op = q.enqueue(counter)
sess = tf.Session()
qr = tf.train.QueueRunner(q, enqueue_ops=[add_op, enqueueData_op] * 2)
sess.run(tf.global_variables_initializer())
enqueue_threads = qr.create_threads(sess, start=True) # 启动入队线程
for i in range(10):
print(sess.run(q.dequeue()))
结果：
会话没有报错，程序也没有结束，而是被挂起，造成这种情况的原因是add操作和入队操作没有同步。
2.0
2.0
4.0
5.0
5.0
5.0
6.0
7.0
8.0
9.0
程序10-4：
import tensorflow as tf
q = tf.FIFOQueue(1000, "float32")
counter = tf.Variable(0.0)
add_op = tf.assign_add(counter, tf.constant(1.0))
enqueueData_op = q.enqueue(counter)
sess = tf.Session()
qr = tf.train.QueueRunner(q, enqueue_ops=[add_op, enqueueData_op] * 2)
sess.run(tf.global_variables_initializer())
enqueue_threads = qr.create_threads(sess, start=True)
coord = tf.train.Coordinator()
enqueue_threads = qr.create_threads(sess, coord=coord, start=True)#添加了一个新的参数，线程协调器，用于协调线程之间的关系
for i in range(0, 10):
print(sess.run(q.dequeue()))
coord.request_stop()
coord.join(enqueue_threads)
结果：
会话依然没有报错，程序也没有结束。
2.0
4.0
6.0
6.0
6.0
8.0
8.0
10.0
10.0
12.0
CSV文件的创建
程序10-5：
1.代码同级路径下新建名为jpg的文件夹，放入十张jpg图片：
2.test0712.py
import os
path = 'jpg'#文件夹的路径
filenames = os.listdir(path)#读取文件路径
strText = ""#字符串，供CSV文件写入使用
with open("train_list.csv", "w") as fid:
for a in range(len(filenames)):
strText = path + os.sep + filenames[a] + "," + filenames[a].split('_')[0] + "\n"
fid.write(strText)
fid.close()
结果：
代码同级路径下生成train_list.csv文件：
程序10-6：
#CSV文件的读取
import tensorflow as tf
import cv2
image_add_list = []#地址数组
image_label_list = []#标签数组
with open("train_list.csv") as fid:
for image in fid.readlines():#读取数据
image_add_list.append(image.strip().split(",")[0])
image_label_list.append(image.strip().split(",")[1])
#图片转化成张量格式
img = tf.image.convert_image_dtype(tf.image.decode_jpeg(tf.read_file('jpg\\image_0001.jpg'), channels=1) , dtype=tf.float32)
print(img)
结果：
Tensor("convert_image:0", shape=(?, ?, 1), dtype=float32)
程序10-7：
import tensorflow as tf
import cv2
image_add_list = []
image_label_list = []
with open("train_list.csv") as fid:
for image in fid.readlines():
image_add_list.append(image.strip().split(",")[0])
image_label_list.append(image.strip().split(",")[1])
# def get_image(image_path):
# return tf.image.convert_image_dtype(
# tf.image.decode_jpeg(
# tf.read_file(image_path), channels=1),
# dtype=tf.uint8)
img = tf.image.convert_image_dtype(tf.image.decode_jpeg(tf.read_file('jpg\\image_0004.jpg'), channels=1),dtype=tf.float32)
with tf.Session() as sess:
cv2Img = sess.run(img)
img2 = cv2.resize(cv2Img, (200,200))
cv2.imshow('image', img2)
cv2.waitKey()
结果：
TensorFlow文件的创建和读取
程序10-8：
import tensorflow as tf
import numpy as np
a_data = 0.834#float类型
b_data = [17]#列表
c_data = np.array([[0, 1, 2], [3, 4, 5]])
c = c_data.astype(np.uint8)
c_raw = c.tostring() # 转化成字符串
example = tf.train.Example(
features=tf.train.Features(
feature={
'a': tf.train.Feature(
float_list=tf.train.FloatList(value=[a_data]) # 方括号表示输入为list
),
'b': tf.train.Feature(
int64_list=tf.train.Int64List(value=b_data) # b_data本身就是列表
),
'c': tf.train.Feature(
bytes_list=tf.train.BytesList(value=[c_raw]) # c_raw被转化成byte格式 设置成二进制的形式进行写入
)
}
)
)
程序10-9：
#随机生成一个数据并将其保存为FRecords
import tensorflow as tf
import numpy as np
# writer = tf.python_io.TFRecordWriter("trainArray.tfrecords")
writer = tf.io.TFRecordWriter("trainArray.tfrecords")
for _ in range(100):
randomArray = np.random.random((1, 3))#生成随机数组
array_raw = randomArray.tobytes()#将数组转化成二进制形式
example = tf.train.Example(features=tf.train.Features(feature={
"label": tf.train.Feature(int64_list=tf.train.Int64List(value=[0])),
'img_raw': tf.train.Feature(bytes_list=tf.train.BytesList(value=[array_raw]))
}))
writer.write(example.SerializeToString())
writer.close()
结果：
在代码同级目录下生成trainArray.tfrecords文件。
图片文件的创建于读取
将图片文件转换成tfrecord文件：
参考链接：
https://www.cnblogs.com/xiaochouk/p/8685909.html
图片文件路径：
test0712.py程序：
import matplotlib.image as mpimg
import tensorflow as tf
def get_label_from_filename(filename):
return 1
filenames = tf.train.match_filenames_once('E:/PycharmProjects/test20190712/jpg0/*.jpg')
writer = tf.io.TFRecordWriter('E:/PycharmProjects/test20190712/train000.tfrecords')
with tf.Session() as sess: #使用match_filenames_once函数需要用tf.local_variables_initializer()函数来实现变量的初始化
sess.run([tf.global_variables_initializer(),tf.local_variables_initializer()])
filenames=(sess.run(filenames))
print(filenames)
#获取的字符串为前面带b：bytes的字符串，类似于字符串前带u：unicode的字符串
#其中从字符串转化成unicode编码的过称为：str.decode('utf-8')，从unicode转化成字符串为：str.encode('utf-8')，因此对如下做同样操作
for filename in filenames:
img=mpimg.imread(filename.decode('utf-8'))
print("{} shape is {}".format(filename, img.shape))
# img_raw = img.tostring()
img_raw = img.tobytes()
label = get_label_from_filename(filename)
example = tf.train.Example(features=tf.train.Features(feature={
"label": tf.train.Feature(int64_list=tf.train.Int64List(value=[label])),
"image_raw": tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw]))
}))
writer.write(record=example.SerializeToString()) #序列转换成字符串
writer.close()
结果：
[b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0000.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0001.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0002.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0003.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0004.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0005.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0006.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0007.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0008.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0009.jpg'
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0010.jpg']
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0000.jpg' shape is (220, 220, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0001.jpg' shape is (436, 414, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0002.jpg' shape is (499, 452, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0003.jpg' shape is (625, 640, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0004.jpg' shape is (220, 220, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0005.jpg' shape is (300, 533, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0006.jpg' shape is (220, 156, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0007.jpg' shape is (918, 1632, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0008.jpg' shape is (583, 500, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0009.jpg' shape is (220, 220, 3)
b'E:\\PycharmProjects\\test20190712\\jpg0\\image_0010.jpg' shape is (220, 220, 3)
代码同级路径下生成train000.tfrecords文件：
程序10-10：
import os
import tensorflow as tf
from PIL import Image
path = "jpg"
filenames=os.listdir(path)
writer = tf.io.TFRecordWriter("train.tfrecords")
#从文件夹中取出下一层的文件夹名和每个文件夹中的图片文件名称
for name in os.listdir(path):
class_path = path + os.sep + name
for img_name in os.listdir(class_path):
img_path = class_path+os.sep+img_name
img = Image.open(img_path)
img = img.resize((500,500))#改变大小
img_raw = img.tobytes()#以字符串形式进行存储
example = tf.train.Example(features=tf.train.Features(feature={
"label": tf.train.Feature(int64_list=tf.train.Int64List(value=[name])),
'image': tf.train.Feature(bytes_list=tf.train.BytesList(value=[img_raw]))
}))
writer.write(example.SerializeToString())
结果报错：
Traceback (most recent call last):
File "E:/PycharmProjects/test20190712/test0712.py", line 19, in <module>
"label": tf.train.Feature(int64_list=tf.train.Int64List(value=[name])),
TypeError: '001' has type str, but expected one of: int, long
程序10-11：
#tfrecords文件读取
import tensorflow as tf
filename = "train.tfrecords"
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue) #读取文件，返回文件名和文件
features = tf.parse_single_example(serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image' : tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['image'], tf.uint8)
img = tf.reshape(img, [300, 300,3])
img = tf.cast(img, tf.float32) * (1. / 128) - 0.5
label = tf.cast(features['label'], tf.int32)
程序10-12：
#tfrecords文件读取，最终生成图片
import tensorflow as tf
import cv2
filename = "train000.tfrecords"
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue) #返回文件名和文件
features = tf.parse_single_example(serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image' : tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['image'], tf.uint8)
img = tf.reshape(img, [300, 300,3])
sess = tf.Session()
init = tf.global_variables_initializer()
sess.run(init)
threads = tf.train.start_queue_runners(sess=sess)
img = tf.cast(img, tf.float32) * (1. / 128) - 0.5
label = tf.cast(features['label'], tf.int32)
imgcv2 = sess.run(img)
cv2.imshow("cool",imgcv2)
cv2.waitKey()
结果报错：
程序不报错，也不停止。
Instructions for updating:
To construct input pipelines, use the `tf.data` module.
程序10-13：
import tensorflow as tf
import cv2
def read_and_decode(filename):
filename_queue = tf.train.string_input_producer([filename])
reader = tf.TFRecordReader()
_, serialized_example = reader.read(filename_queue) # 返回文件名和文件
features = tf.parse_single_example(serialized_example,
features={
'label': tf.FixedLenFeature([], tf.int64),
'image': tf.FixedLenFeature([], tf.string),
})
img = tf.decode_raw(features['image'], tf.uint8)
img = tf.reshape(img, [300, 300, 3])
img = tf.cast(img, tf.float32) * (1. / 128) - 0.5
label = tf.cast(features['label'], tf.int32)
return img,label
filename = "train.tfrecords"
img,label = read_and_decode(filename)
img_batch,label_batch = tf.train.shuffle_batch([img,label],batch_size=1,
capacity=10,
min_after_dequeue=1)
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
threads = tf.train.start_queue_runners(sess=sess)
for _ in range(10):
val = sess.run(img_batch)
label = sess.run(label_batch)
val.resize((300,300,3))
cv2.imshow("cool",val)
cv2.waitKey()
print(label)
结果报错：
结果和10-12一样
3610
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
