kafka Expiring 536 record(s) for ka1-0: 30009 ms has passed since last append

kafka 大批量发送消息时报
the producer has a error:Expiring 536 record(s) for ka1-0: 30009 ms has passed since last append
 Expiring 536 record(s) for ka1-0: 30009 ms has passed since last append

有其他网友说可能是topic不是手动通过脚本创建的，所以导致消息发送失败，但是我的topic是自己手动创建的也报这个异常，后来我怀疑是不是提交的太快了，每次producer.send完我sleep一个毫秒，虽然降低了发送速率，但是没有报异常了，4个线程同时发送，每个线程到目前为止发送了200万条记录，没有任何异常。大批量发送至于为什么报那个异常我到现在也没有找到原因，如果有网友找到解决方案，请告知一下，谢谢。

-----------------------------------------------------

刚刚看了一下源码，有一些自己的了解和大家分享一下，也不知道对不对。

requestTimeoutMs 默认值是30秒，this.lastAppendTime是构造ProducerBatch创建时传入的时间，见代码片段二；而now 我这里跟踪到是线程启动的时获取的时间，见代码片段三。
因为我这是客户端死循环不停的调用producer.sende函数，一直发送。当如果处理到这个ProducerBatch 可能时间差已经超过30秒（代码片段一），导致这个producerBatch过期。可以增大requestTimeoutMs 值降低过期ProducerBatch 的数量，但不是最好的解决方案，继续看看是否有更好的方式。

 代码片段一

    boolean maybeExpire(int requestTimeoutMs, long retryBackoffMs, long now, long lingerMs, boolean isFull) {
        if (!this.inRetry() && isFull && requestTimeoutMs < (now - this.lastAppendTime))
            expiryErrorMessage = (now - this.lastAppendTime) + " ms has passed since last append";
        else if (!this.inRetry() && requestTimeoutMs < (createdTimeMs(now) - lingerMs))
            expiryErrorMessage = (createdTimeMs(now) - lingerMs) + " ms has passed since batch creation plus linger time";
        else if (this.inRetry() && requestTimeoutMs < (waitedTimeMs(now) - retryBackoffMs))
            expiryErrorMessage = (waitedTimeMs(now) - retryBackoffMs) + " ms has passed since last attempt plus backoff time";


        boolean expired = expiryErrorMessage != null;
        if (expired)
            abortRecordAppends();
        return expired;
    }


代码片段二
public ProducerBatch(TopicPartition tp, MemoryRecordsBuilder recordsBuilder, long now) {
        this(tp, recordsBuilder, now, false);
    }


    public ProducerBatch(TopicPartition tp, MemoryRecordsBuilder recordsBuilder, long now, boolean isSplitBatch) {
        this.createdMs = now;
        this.lastAttemptMs = now;
        this.recordsBuilder = recordsBuilder;
        this.topicPartition = tp;
        this.lastAppendTime = createdMs;
        this.produceFuture = new ProduceRequestResult(topicPartition);
        this.retry = false;
        this.isSplitBatch = isSplitBatch;
        float compressionRatioEstimation = CompressionRatioEstimator.estimation(topicPartition.topic(),
                                                                                recordsBuilder.compressionType());
        recordsBuilder.setEstimatedCompressionRatio(compressionRatioEstimation);
    }


代码片段三
/**
     * The main run loop for the sender thread
     */
    public void run() {
        log.debug("Starting Kafka producer I/O thread.");


        // main loop, runs until close is called
        while (running) {
            try {
                run(time.milliseconds());
            } catch (Exception e) {
                log.error("Uncaught error in kafka producer I/O thread: ", e);
            }
        }


        log.debug("Beginning shutdown of Kafka producer I/O thread, sending remaining records.");


        // okay we stopped accepting requests but there may still be
        // requests in the accumulator or waiting for acknowledgment,
        // wait until these are completed.
        while (!forceClose && (this.accumulator.hasUndrained() || this.client.inFlightRequestCount() > 0)) {
            try {
                run(time.milliseconds());
            } catch (Exception e) {
                log.error("Uncaught error in kafka producer I/O thread: ", e);
            }
        }

       //........
    }