Tensorflow调用tf.train.shuffle_batch函数报错“OutOfRangeError: RandomShuffleQueue”

1.报错内容:

OutOfRangeError (see above for traceback): RandomShuffleQueue '_1_shuffle_batch/random_shuffle_queue' is closed and has insufficient elements (requested 3, current size 0) [[Node: shuffle_batch = QueueDequeueManyV2[component_types=[DT_FLOAT, DT_FLOAT], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/device:CPU:0"](shuffle_batch/random_shuffle_queue, shuffle_batch/n)]]

2.问题说明

首先,初始该类问题表明程序本身并没有实质性的语法错误,其次,该报错是由shuffle_batch函数生成的

3.错误排查

(1)若显示:“requested 3, current size 1”表明数据还未读入,可尝试调小batch_size(随机出队列的数据量); 或者增大num_threads(线程数量)

(2)若显示:“requested 3, current size 1”则表示数据本身并没有读入,则可检查数据集是否有问题

比如:

①一个对象(行)应当有7个元素进行表示,却只有6个元素)

②按照指定格式读取数据,如[[1.0], [1.0], [1.0]], 却读取了数据标题行的字符型标题,导致数据无法读入,解决的方法有两个:①tf.TextLineReader()等函数读取数据时直接跳过第一行;②在原始数据中删除第一行,即删除第一行标题行

4.示例代码

import tensorflow as tf
import numpy as np
import os
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"


def readFile(filename):
    filename_queue = tf.train.string_input_producer(filename, shuffle=False)

    # 定义reader(逐行读取)
    reader = tf.TextLineReader(skip_header_lines=1)
    key, value = reader.read(filename_queue)

    # 读取数据时对应的匹配类型
    # [1.0]:float
    # [1]:int
    # ["null"]:string
    record_defaults = [[1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0]]
    # record_defaults = []
    # for i in range(7):
    #     record_defaults.append([1.0])
    col1, col2, col3, col4, col5, col6, col7 = tf.decode_csv(value, record_defaults=record_defaults)

    # tf.stack:按照给定方向堆叠
    label = tf.stack([col1, col2])
    features = tf.stack([col3, col4, col5, col6, col7])

    # 注意min_after_dequeue < capacity
    example_batch, label_batch = tf.train.shuffle_batch([features, label], batch_size=3, capacity=100, min_after_dequeue=10)
    return example_batch, label_batch


example_batch, label_batch = readFile(["cancer.txt"])

with tf.Session() as sess:
    
    # 数据读取必须要创建线程协调器
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    for i in range(5):
        e_val, l_val = sess.run([example_batch, label_batch])
        print(e_val)
    coord.request_stop()
    coord.join(threads)

 

import tensorflow as tf from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPool2D, Dropoutfrom tensorflow.keras import Model​# 在GPU上运算时,因为cuDNN库本身也有自己的随机数生成器,所以即使tf设置了seed,也不会每次得到相同的结果tf.random.set_seed(100)​mnist = tf.keras.datasets.mnist(X_train, y_train), (X_test, y_test) = mnist.load_data()X_train, X_test = X_train/255.0, X_test/255.0​# 将特征数据集从(N,32,32)转变成(N,32,32,1),因为Conv2D需要(NHWC)四阶张量结构X_train = X_train[..., tf.newaxis]    X_test = X_test[..., tf.newaxis]​batch_size = 64# 手动生成mini_batch数据集train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(10000).batch(batch_size)test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(batch_size)​class Deep_CNN_Model(Model):    def __init__(self):        super(Deep_CNN_Model, self).__init__()        self.conv1 = Conv2D(32, 5, activation='relu')        self.pool1 = MaxPool2D()        self.conv2 = Conv2D(64, 5, activation='relu')        self.pool2 = MaxPool2D()        self.flatten = Flatten()        self.d1 = Dense(128, activation='relu')        self.dropout = Dropout(0.2)        self.d2 = Dense(10, activation='softmax')        def call(self, X):    # 无需在此处增加training参数状态。只需要在调用Model.call时,传递training参数即可        X = self.conv1(X)        X = self.pool1(X)        X = self.conv2(X)        X = self.pool2(X)        X = self.flatten(X)        X = self.d1(X)        X = self.dropout(X)   # 无需在此处设置training状态。只需要在调用Model.call时,传递training参数即可        return self.d2(X)​model = Deep_CNN_Model()loss_object = tf.keras.losses.SparseCategoricalCrossentropy()optimizer = tf.keras.optimizers.Adam()​train_loss = tf.keras.metrics.Mean(name='train_loss')train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy')test_loss = tf.keras.metrics.Mean(name='test_loss')test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy')​# TODO:定义单批次的训练和预测操作@tf.functiondef train_step(images, labels):       ......    @tf.functiondef test_step(images, labels):       ......    # TODO:执行完整的训练过程EPOCHS = 10for epoch in range(EPOCHS)补全代码
06-13
import tensorflow as tf from tensorflow.keras.layers import Dense, Flatten, Conv2D, MaxPool2D, Dropout from tensorflow.keras import Model # 在GPU上运算时,因为cuDNN库本身也有自己的随机数生成器,所以即使tf设置了seed,也不会每次得到相同的结果 tf.random.set_seed(100) # 加载MNIST数据集并进行预处理 mnist = tf.keras.datasets.mnist (X_train, y_train), (X_test, y_test) = mnist.load_data() X_train, X_test = X_train/255.0, X_test/255.0 X_train = X_train[..., tf.newaxis] X_test = X_test[..., tf.newaxis] # 定义batch_size并手动生成mini_batch数据集 batch_size = 64 train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(10000).batch(batch_size) test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(batch_size) # 定义深度CNN模型 class Deep_CNN_Model(Model): def __init__(self): super(Deep_CNN_Model, self).__init__() self.conv1 = Conv2D(32, 5, activation='relu') self.pool1 = MaxPool2D() self.conv2 = Conv2D(64, 5, activation='relu') self.pool2 = MaxPool2D() self.flatten = Flatten() self.d1 = Dense(128, activation='relu') self.dropout = Dropout(0.2) self.d2 = Dense(10, activation='softmax') def call(self, X, training=False): X = self.conv1(X) X = self.pool1(X) X = self.conv2(X) X = self.pool2(X) X = self.flatten(X) X = self.d1(X) X = self.dropout(X, training=training) return self.d2(X) # 实例化模型,定义损失函数和优化器 model = Deep_CNN_Model() loss_object = tf.keras.losses.SparseCategoricalCrossentropy() optimizer = tf.keras.optimizers.Adam() # 定义训练和测试过程中的度量指标 train_loss = tf.keras.metrics.Mean(name='train_loss') train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') test_loss = tf.keras.metrics.Mean(name='test_loss') test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy') # 定义单批次的训练和预测操作 @tf.function def train_step(images, labels): with tf.GradientTape() as tape: # 计算模型预测结果和损失函数值 predictions = model(images, training=True) loss = loss_object(labels, predictions) # 计算梯度并优化模型参数 gradients = tape.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) # 更新训练过程中的度量指标 train_loss(loss) train_accuracy(labels, predictions) @tf.function def test_step(images, labels): # 计算模型预测结果和损失函数值 predictions = model(images, training=False) t_loss = loss_object(labels, predictions) # 更新测试过程中的度量指标 test_loss(t_loss) test_accuracy(labels, predictions) # 执行完整的训练过程 EPOCHS = 10 for epoch in range(EPOCHS): # 在每轮训练开始时,重置度量指标的值 train_loss.reset_states() train_accuracy.reset_states() test_loss.reset_states() test_accuracy.reset_states() # 遍历训练数据集中的每个mini_batch进行训练 for images, labels in train_ds: train_step(images, labels) # 遍历测试数据集中的每个mini_batch进行预测 for test_images, test_labels in test_ds: test_step(test_images, test_labels) # 输出每轮训练和测试过程中的度量指标 template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}' print(template.format(epoch+1, train_loss.result(), train_accuracy.result()*100, test_loss.result(), test_accuracy.result()*100))
评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包

打赏作者

博士僧小星

你的鼓励将是我创作的最大动力

¥1 ¥2 ¥4 ¥6 ¥10 ¥20
扫码支付:¥1
获取中
扫码支付

您的余额不足,请更换扫码支付或充值

打赏作者

实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值