python堆栈跟踪_python - CTCBeamSearchDecoder - 在波束搜索中留下的请求少于请求 - 堆栈内存溢出...

在成功培训了多批数据（每次都有不同的数字）后，我遇到了这个问题。 有趣的是，如果我只对发生错误的批处理进行训练，一切都很好 - 所以数据没有任何问题（我正在使用Tensorflow speech_commands数据集）。 我对导致这种行为的原因感到很困惑。 任何帮助是极大的赞赏。

堆栈跟踪：

Traceback (most recent call last):

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1334, in _do_call

return fn(*args)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1319, in _run_fn

options, feed_dict, fetch_list, target_list, run_metadata)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun

run_metadata)

tensorflow.python.framework.errors_impl.InvalidArgumentError: Less leaves in the beam search than requested.

[[{{node loss/CTCBeamSearchDecoder}} = CTCBeamSearchDecoder[beam_width=100, merge_repeated=true, top_paths=2, _device="/job:localhost/replica:0/task:0/device:CPU:0"](loss/transpose_1/_91, Fill/_93)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):

File "train.py", line 180, in

train_and_eval()

File "train.py", line 107, in train_and_eval

feed_dict=feed_dict)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 929, in run

run_metadata_ptr)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1152, in _run

feed_dict_tensor, options, run_metadata)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1328, in _do_run

run_metadata)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/client/session.py", line 1348, in _do_call

raise type(e)(node_def, op, message)

tensorflow.python.framework.errors_impl.InvalidArgumentError: Less leaves in the beam search than requested.

[[node loss/CTCBeamSearchDecoder (defined at /mainfs/lyceum/chk1g16/Speech-Recognition/Conv_LSTM_CTC/conv_lstm_ctc_net.py:256) = CTCBeamSearchDecoder[beam_width=100, merge_repeated=true, top_paths=2, _device="/job:localhost/replica:0/task:0/device:CPU:0"](loss/transpose_1/_91, Fill/_93)]]

Caused by op 'loss/CTCBeamSearchDecoder', defined at:

File "train.py", line 180, in

train_and_eval()

File "train.py", line 55, in train_and_eval

data_gen._num_frames, data_gen._num_mel_spec_bins, init_lr, lr_decay_steps, lr_decay_rate)

File "/mainfs/lyceum/chk1g16/Speech-Recognition/Conv_LSTM_CTC/conv_lstm_ctc_net.py", line 330, in create_train_graph

predictions, loss, acc_greedy, edit_dist_greedy, acc_beam, edit_dist_beam, scores = get_ctc_loss(logits, label_batch_plh)

File "/mainfs/lyceum/chk1g16/Speech-Recognition/Conv_LSTM_CTC/conv_lstm_ctc_net.py", line 256, in get_ctc_loss

merge_repeated=True)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/ops/ctc_ops.py", line 277, in ctc_beam_search_decoder

merge_repeated=merge_repeated))

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/ops/gen_ctc_ops.py", line 74, in ctc_beam_search_decoder

top_paths=top_paths, merge_repeated=merge_repeated, name=name)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper

op_def=op_def)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py", line 488, in new_func

return func(*args, **kwargs)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3274, in create_op

op_def=op_def)

File "/lyceum/chk1g16/.conda/envs/py3venv/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1770, in __init__

self._traceback = tf_stack.extract_stack()

InvalidArgumentError (see above for traceback): Less leaves in the beam search than requested.

[[node loss/CTCBeamSearchDecoder (defined at /mainfs/lyceum/chk1g16/Speech-Recognition/Conv_LSTM_CTC/conv_lstm_ctc_net.py:256) = CTCBeamSearchDecoder[beam_width=100, merge_repeated=true, top_paths=2, _device="/job:localhost/replica:0/task:0/device:CPU:0"](loss/transpose_1/_91, Fill/_93)]]

损失函数：

def get_ctc_loss(logits, label_batch):

# logits: [batch_size, max_time, num_classes]

# 1-D tensor showing the length for each label in the batch

batch_labels_lengths = tf.fill([tf.shape(label_batch)[0]], tf.shape(label_batch)[1])

with tf.name_scope('loss'):

# get sparse represenattion of the labels

non_zero_elems_coords = tf.where(tf.not_equal(label_batch, 0))

non_zero_elems = tf.gather_nd(label_batch, non_zero_elems_coords)

sparse_label_batch = tf.SparseTensor(indices=non_zero_elems_coords,

values=non_zero_elems,

dense_shape=tf.shape(label_batch, out_type=tf.int64))

# calculate ctc loss

ctc_loss_op = tf.nn.ctc_loss(labels=sparse_label_batch,

inputs=logits,

sequence_length=batch_labels_lengths,

preprocess_collapse_repeated=True,

time_major=False,

ctc_merge_repeated=True,

ignore_longer_outputs_than_inputs=False)

loss = tf.reduce_mean(ctc_loss_op)

prediction_probabilities = tf.nn.softmax(logits)

max_probabilities = tf.reduce_max(prediction_probabilities, axis=2)

raw_predictions = tf.argmax(prediction_probabilities, axis=2)

# greedy decode logits

# greedy decoder - beeam decoder with beam_width=1 and top_paths=1

logits_T = tf.transpose(logits, perm=[1, 0, 2])

greedy_predictions, neg_sum_logits = tf.nn.ctc_greedy_decoder(inputs=logits_T,

sequence_length=batch_labels_lengths,

merge_repeated=True)

# get greedy performance metrics

edit_dist_greedy = tf.edit_distance(tf.cast(greedy_predictions[0], tf.int32),

sparse_label_batch,

normalize=False)

acc_greedy = tf.reduce_mean(tf.cast(tf.equal(edit_dist_greedy, 0), tf.float32))

edit_dist_greedy = tf.reduce_mean(edit_dist_greedy)

# beam decode logits

beam_predictions, log_probabilities = tf.nn.ctc_beam_search_decoder(inputs=logits_T,

sequence_length=batch_labels_lengths,

beam_width=100,

top_paths=2,

merge_repeated=True)

# get beam performance metrics

edit_dist_beam = tf.edit_distance(tf.cast(beam_predictions[0], tf.int32),

sparse_label_batch,

normalize=False)

acc_beam = tf.reduce_mean(tf.cast(tf.equal(edit_dist_beam, 0), tf.float32))

predictions = tf.cast(tf.sparse.to_dense(beam_predictions[0]), tf.int32)

scores = log_probabilities[:, 0] - log_probabilities[:, 1]

tf.summary.scalar('ctc_loss', loss)

tf.summary.scalar('acc_greedy', acc_greedy)

tf.summary.scalar('edit_dist_greedy', edit_dist_greedy)

tf.summary.scalar('confidence_score', tf.reduce_mean(scores))

# tf.summary.scalar('edit_dist_beam', edit_dist_beam)

tf.summary.scalar('acc_beam', acc_beam)

return predictions, loss, acc_greedy, edit_dist_greedy, acc_beam, edit_dist_beam, scores