Kafka producer源码解析

KafkaProducer

 从 procuder.send 说起

         try {
              val kafkaProducerRecord =new ProducerRecord[String, String]("live_order_id_info_back", sedMsg)
              procuder.send(kafkaProducerRecord)
            } catch {
              case _: Exception => procuder.close()
            }

 调用的是 KafkaProducer构造函数, 调用send发送(其实都是异步处理) ----> 调用的doSend()

 

 public Future<RecordMetadata> send(ProducerRecord<K, V> record) {
        return this.send(record, (Callback)null);
    }

    public Future<RecordMetadata> send(ProducerRecord<K, V> record, Callback callback) {
        ProducerRecord<K, V> interceptedRecord = this.interceptors.onSend(record);
        return this.doSend(interceptedRecord, callback);
    }

    private void throwIfProducerClosed() {
        if (this.ioThread == null || !this.ioThread.isAlive()) {
            throw new IllegalStateException("Cannot perform operation after producer has been closed");
        }
    }

    private Future<RecordMetadata> KafkaProducer(ProducerConfig config, Serializer<K> keySerializer, Serializer<V> valueSerializer)  {
        try {
            log.trace("Starting the Kafka producer");
            Map<String, Object> userProvidedConfigs = config.originals();
            this.producerConfig = config;
            this.time = new SystemTime();

            clientId = config.getString(ProducerConfig.CLIENT_ID_CONFIG);
            //配置中解析出clientId,用于跟踪程序运行情况,在有多个KafkProducer时,若没有配置 client.id则clientId 以前 辍”producer-”后加一个从 1 递增的整数
            if (clientId.length() <= 0)
                clientId = "producer-" + PRODUCER_CLIENT_ID_SEQUENCE.getAndIncrement();
            //注册用于Kafka metrics指标收集的相关对象,用于对 Kafka 集群相关指标的追踪
            Map<String, String> metricTags = new LinkedHashMap<String, String>();
            metricTags.put("client-id", clientId);
            MetricConfig metricConfig = new MetricConfig().samples(config.getInt(ProducerConfig.METRICS_NUM_SAMPLES_CONFIG))
                    .timeWindow(config.getLong(ProducerConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS)
                    .tags(metricTags);
            List<MetricsReporter> reporters = config.getConfiguredInstances(ProducerConfig.METRIC_REPORTER_CLASSES_CONFIG,
                    MetricsReporter.class);
            reporters.add(new JmxReporter(JMX_PREFIX));
            this.metrics = new Metrics(metricConfig, reporters, time);
            //初始化分区选择器 通過反射獲取
            this.partitioner = config.getConfiguredInstance(ProducerConfig.PARTITIONER_CLASS_CONFIG, Partitioner.class);
            long retryBackoffMs = config.getLong(ProducerConfig.RETRY_BACKOFF_MS_CONFIG);
            //初始集群元数据、消息缓冲区大小、压缩策略
            this.metadata = new Metadata(retryBackoffMs, config.getLong(ProducerConfig.METADATA_MAX_AGE_CONFIG));
            this.maxRequestSize = config.getInt(ProducerConfig.MAX_REQUEST_SIZE_CONFIG);
            this.totalMemorySize = config.getLong(ProducerConfig.BUFFER_MEMORY_CONFIG);
            this.compressionType = CompressionType.forName(config.getString(ProducerConfig.COMPRESSION_TYPE_CONFIG));
       
            //实例化用于存储消息的RecordAccumulator,作用类似一个队列 
            //指定每个RecordBatch的大小，单位是字节
            this.accumulator = new RecordAccumulator(config.getInt(ProducerConfig.BATCH_SIZE_CONFIG),
                    this.totalMemorySize,
                    this.compressionType,
                    config.getLong(ProducerConfig.LINGER_MS_CONFIG),
                    retryBackoffMs,