Kafka 实例

首先引入相关jar

         <dependency>
            <groupId>org.apache.kafka</groupId>
            <artifactId>kafka-clients</artifactId>
            <version>0.10.0.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.zookeeper</groupId>
            <artifactId>zookeeper</artifactId>
            <version>3.4.6</version>
        </dependency>
        <dependency>
            <groupId>com.101tec</groupId>
            <artifactId>zkclient</artifactId>
            <version>0.9</version>
        </dependency>

生产者

/**
 * sunjiamin@sunjiamin
 *
 * @author sunjiamin
 * @version 0.0.1
 * @desc double check实现KafkaProducer的懒加载
 * @date 2017-09-07 09:02:34
 */
public class LazySingletonProducer {

    private static volatile Producer<byte[], String> producer;

    /**
     * 私有化构造方法
     */
    private LazySingletonProducer() {

    }

    /**
     * 实例化
     * @param config
     * @return
     */
    public static Producer<byte[], String> getInstance(Map<String, Object> config) {
        if (producer == null) {
            synchronized(LazySingletonProducer.class) {
                if (producer == null) {
                    producer = new KafkaProducer<byte[], String>(config);
                }
            }
        }
        return producer;
    }

    /**
     * 是否初始化
     * @return
     */
    public static boolean isInstanced() {
        return producer != null;
    }
}

客户端通过Producer发送消息

/**
     * 向kafka send
     * @param value
     */
    private void send(String value) {
        final byte[] key = ByteBuffer.allocate(4).putInt(new StringBuilder(app).append(host).toString().hashCode()).array();

        final ProducerRecord<byte[], String> record = new ProducerRecord<byte[], String>(this.topic, key, value);
        LazySingletonProducer.getInstance(this.config).send(record, new Callback() {
            public void onCompletion(RecordMetadata recordMetadata, Exception e) {
                // TODO: 异常发生如何处理(直接停掉appender)

            }
        });
    }

消费者：

构建消费者bean

@Configuration
@EnableConfigurationProperties({KafkaProperties.class})
public class KafkaConfiguration {

    @Autowired
    private KafkaProperties kafkaProperties;

    // kafka consumer
    @Bean
    public KafkaConsumer<byte[], String> kafkaConsumer() {
        Map<String, Object> config = new HashMap<String, Object>();
        config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, this.kafkaProperties.getBrokers());
        config.put(ConsumerConfig.GROUP_ID_CONFIG, this.kafkaProperties.getConsumeGroup());
        // 手动commit offset到kafka(该版本不将offset保存到zk)
        config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        config.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 15000);
        config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
        config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

        KafkaConsumer<byte[], String> kafkaConsumer = new KafkaConsumer<byte[], String>(config);

        return kafkaConsumer;
    }

}

引用的属性配置文件：

@ConfigurationProperties(prefix = "spring.message.kafka")
public class KafkaProperties {

    private String brokers;

    private String consumeGroup;

    private long pollTimeout;

    private String topic;

    public String getBrokers() {
        return brokers;
    }

    public void setBrokers(String brokers) {
        this.brokers = brokers;
    }

    public String getConsumeGroup() {
        return consumeGroup;
    }

    public void setConsumeGroup(String consumeGroup) {
        this.consumeGroup = consumeGroup;
    }

    public long getPollTimeout() {
        return pollTimeout;
    }

    public void setPollTimeout(long pollTimeout) {
        this.pollTimeout = pollTimeout;
    }

    public String getTopic() {
        return topic;
    }

    public void setTopic(String topic) {
        this.topic = topic;
    }
}

消费者客户端消费：

@Autowired
    private KafkaConsumer kafkaConsumer;
    @Autowired
    private KafkaProperties kafkaProperties;
    @Autowired
    private FileUtil fileUtil;

    public static Map<TopicPartition, OffsetAndMetadata> currentOffsets = new HashMap<TopicPartition, OffsetAndMetadata>();
    private Thread thread;

    @Override
    public void doTask() {
        this.thread = Thread.currentThread();
        int count = 0;
        try {
            while (true) {
                ConsumerRecords<byte[], String> records = this.kafkaConsumer.poll(this.kafkaProperties.getPollTimeout());
                if (!records.isEmpty()) {
                    Map<String, List<String>> lines = new HashMap<String, List<String>>();
                    for (ConsumerRecord<byte[], String> record : records) {
                        String value = record.value();
                        LogDto logDto = this.getLogDto(value);
                        if (logDto != null) {
                            if (LogLevel.INFO.isLegal(logDto.getLevel())) {
                                // 是info、error或者warning的日志才进行处理
                                String key = logDto.getDay();
                                if (lines.containsKey(key)) {
                                    // 如果已经存在此天的数据
                                    lines.get(key).add(value);
                                } else {
                                    // 不存在此天的数据
                                    List<String> tmpLines = new ArrayList<String>();
                                    tmpLines.add(value);
                                    lines.put(key, tmpLines);
                                }
                            }
                        } else {
                            LOGGER.info("record transform error, {}", value);
                        }
                        currentOffsets.put(new TopicPartition(record.topic(), record.partition()), new OffsetAndMetadata(record.offset() + 1));

                        count++;
                        if (count >= 1000) {
                            // 当达到了1000触发向kafka提交offset
                            this.kafkaConsumer.commitAsync(currentOffsets, new KafkaOffsetCommitCallback());
                            count = 0;
                        }
                    }
                    // save to file
                    int size = this.fileUtil.save(lines);

                    this.kafkaConsumer.commitAsync(currentOffsets, new KafkaOffsetCommitCallback());
                    LOGGER.info("total record: {}, saved {} records to file", records.count(), size);
                }
            }
        } catch (WakeupException e) {
            // do not process, this is shutdown
            LOGGER.error("wakeup, start to shutdown, {}", e);
        } catch (Exception e) {
            LOGGER.error("process records error, {}", e);
        } finally {
            this.kafkaConsumer.commitSync(currentOffsets);
            LOGGER.info("finally commit the offset");
            // 不需要主动调kafkaConsumer.close(), spring bean容器会调用
        }
    }