简单封装kafka相关的api

最新推荐文章于 2022-04-18 21:14:18 发布
最新推荐文章于 2022-04-18 21:14:18 发布
阅读量277 收藏
点赞数
文章标签： 大数据 json java
原文链接：http://www.cnblogs.com/hujunzheng/p/9327927.html
版权

一、针对于kafka版本

<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka-clients</artifactId>
    <version>0.8.2.2</version>

</dependency>

<dependency>
    <groupId>org.apache.kafka</groupId>
    <artifactId>kafka_2.11</artifactId>
    <version>0.8.2.2</version>
</dependency>

二、操作topic、consumer相关方法

import kafka.admin.AdminUtils;
import kafka.admin.TopicCommand;
import kafka.api.TopicMetadata;
import kafka.tools.ConsumerOffsetChecker;
import kafka.utils.ZKStringSerializer$;
import kafka.utils.ZkUtils;
import org.I0Itec.zkclient.ZkClient;
import org.apache.commons.exec.CommandLine;
import org.apache.commons.exec.DefaultExecutor;
import org.apache.commons.exec.ExecuteWatchdog;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.tools.ant.taskdefs.Execute;

public static class KafkaUtils {
    private static Logger LOGGER = LoggerFactory.getLogger(KafkaUtils.class);

    private static AutoZkClient zkClient;

    public static AutoZkClient getZkClient() {
        return zkClient;
    }

    public static void setZkClient(AutoZkClient zkClient) {
        KafkaUtils.zkClient = zkClient;
    }

    public static boolean topicExists(String topic) {
        Assert.notNull(zkClient, "zkclient is null");
        return AdminUtils.topicExists(zkClient, topic);
    }

    public static void topicChangeConfig(String topic, Properties properties) {
        Assert.notNull(zkClient, "zkclient is null");
        AdminUtils.changeTopicConfig(zkClient, topic, properties);
    }

    public static void topicAlterPartitions(String topic, int partitions) {
        Assert.notNull(zkClient, "zkclient is null");
        TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient);
        int curPartitions = topicMetadata.partitionsMetadata().size();
        if (curPartitions == partitions) {
            return;
        }
        if (curPartitions > partitions) {
            LOGGER.info(String.format("curPartitions=%d，不能修改partitions=%d，请确保大与当前分区数", curPartitions, partitions));
            return;
        }
        String[] args = {
                "--zookeeper", zkClient.zkServers,
                "--partitions", String.valueOf(partitions),
                "--alter",
                "--topic", topic
        };
        TopicCommand.TopicCommandOptions alterOpt = new TopicCommand.TopicCommandOptions(args);
        alterOpt.checkArgs();
        TopicCommand.alterTopic(zkClient, alterOpt);
    }

    public static void topicDescribe(String topic) {
        Assert.notNull(zkClient, "zkclient is null");
        String[] args = {
                "--zookeeper", zkClient.zkServers,
                "--describe",
                "--topic", topic
        };
        TopicCommand.TopicCommandOptions describeOpt = new TopicCommand.TopicCommandOptions(args);
        describeOpt.checkArgs();
        TopicCommand.describeTopic(zkClient, describeOpt);
    }

    public static void topicOverrideConfig(String topic, Properties properties) {
        Assert.notNull(zkClient, "zkclient is null");
        Properties oldProperties = KafkaUtils.topicConfig(topic);
        oldProperties.putAll(properties);
        AdminUtils.changeTopicConfig(zkClient, topic, oldProperties);
    }


    public static void topicCreate(TopicConfig topicConfig) {
        Assert.notNull(zkClient, "zkclient is null");
        int brokerSize = ZkUtils.getSortedBrokerList(zkClient).size();
        if (topicConfig.getReplicationFactor() > brokerSize) {
            topicConfig.setReplicationFactor(brokerSize);
            LOGGER.info(String.format("broker-size=%d < replication-factor=%d, 所以设置replication-factor大小为broker-size大小"
                    , brokerSize, topicConfig.getReplicationFactor()));
        }
        AdminUtils.createTopic(zkClient
                , topicConfig.getName()
                , topicConfig.getPartitions()
                , topicConfig.getReplicationFactor()
                , topicConfig.getProperties());
    }

    public static void topicDelete(String topic) {
        Assert.notNull(zkClient, "zkclient is null");
        AdminUtils.deleteTopic(zkClient, topic);
    }

    public static List<String> topicsList() {
        Assert.notNull(zkClient, "zkclient is null");
        return seqAsJavaList(ZkUtils.getAllTopics(zkClient));
    }

    public static Properties topicConfig(String topic) {
        Assert.notNull(zkClient, "zkclient is null");
        return AdminUtils.fetchTopicConfig(zkClient, topic);
    }

    public static Map<String, Properties> topicsConfig() {
        Assert.notNull(zkClient, "zkclient is null");
        return mapAsJavaMap(AdminUtils.fetchAllTopicConfigs(zkClient));
    }

    public static void consumerDetail(String topic, String group){
        String[] args = {
                "--zookeeper", zkClient.getZkServers(),
                "--group", group,
                "--topic", topic
        };
        ConsumerOffsetChecker.main(args);
    }

    public static Map<String, List<String>> getConsumersPerTopic(String group) {
        return mapAsJavaMap(ZkUtils.getConsumersPerTopic(zkClient, group, false)).entrySet()
                .stream().collect(Collectors.toMap(entry -> entry.getKey(), entry -> JavaConversions.seqAsJavaList(entry.getValue())
                        .stream().map(consumerThreadId -> consumerThreadId.consumer())
                        .collect(Collectors.toList())));
    }

    public static List<String> getConsumersInGroup(String group){
        return JavaConversions.seqAsJavaList(ZkUtils.getConsumersInGroup(zkClient, group));
    }

    public static String executeCommond(String commond) {
        LOGGER.info("begin to execute commond: " + commond);
        File tmpFileDir = Files.createTempDir();
        String tmpFileName = UUID.randomUUID().toString() + ".txt";
        String fileSavePath = tmpFileDir.getAbsolutePath() + tmpFileName;
        CommandLine oCmdLine = CommandLine.parse(commond + " > " + fileSavePath);
        DefaultExecutor executor = new DefaultExecutor();
        ExecuteWatchdog watchdog = new ExecuteWatchdog(20000);
        executor.setWatchdog(watchdog);
        int[] exitValues = {0, 1};
        executor.setExitValues(exitValues);
        try {
            if (Execute.isFailure(executor.execute(oCmdLine))) {
                watchdog.killedProcess();
                LOGGER.error("远程命令执行失败... commond=" + commond);
            } else {
                try (Stream<String> lines = java.nio.file.Files.lines(new File(fileSavePath).toPath(), Charset.defaultCharset())) {
                    List<String> fileLines = lines.collect(toCollection(LinkedList::new));
                    StringBuilder result = new StringBuilder();
                    fileLines.forEach(line -> result.append(line).append(System.lineSeparator()));
                    return result.toString();
                } finally {
                    FileUtils.deleteQuietly(tmpFileDir);
                }
            }
        } catch (Exception e) {
            LOGGER.error("execute command error happened... commond=" + commond, e);
        }
        return StringUtils.EMPTY;
    }
}

三、控制层展示

import com.alibaba.fastjson.JSON;
import com.cmos.common.annotation.CompatibleOutput;
import com.cmos.core.logger.Logger;
import com.cmos.core.logger.LoggerFactory;
import com.cmos.wmhopenapi.web.config.KafkaMessageConfig;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.*;
import scala.Console;

import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.util.List;

import static com.cmos.wmhopenapi.web.config.KafkaMessageConfig.KafkaUtils;

/**
 * @author hujunzheng
 * @create 2018-07-16 10:20
 **/
@RestController
@RequestMapping("/message/state")
@CompatibleOutput
public class MessageCenterStateController {
    private static Logger LOGGER = LoggerFactory.getLogger(MessageCenterStateController.class);

    @Autowired
    private KafkaMessageConfig.NoAckConsumer noAckConsumer;

    @Autowired
    private KafkaMessageConfig.AckConsumer ackConsumer;


    /**
     * 获取topic描述
     *
     * @param topic
     **/
    @GetMapping("/topic-describe")
    public String topicDescribe(@RequestParam String topic) {
        try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
            //scala 输出流重定向
            Console.setOut(new PrintStream(bos));
            KafkaUtils.topicDescribe(topic);
            String result = bos.toString();
            LOGGER.info(result);
            return String.format("%s%s%s", "<pre>", result, "</pre>");
        } catch (Exception e) {
            LOGGER.error("获取topic描述异常", e);
        }
        return StringUtils.EMPTY;
    }

    /**
     * 获取全部topic
     **/
    @GetMapping(value = "/topics-all", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
    public String topicAll() {
        String result = JSON.toJSONString(KafkaUtils.topicsList());
        LOGGER.info(result);
        return result;
    }

    /**
     * 获取topic配置
     *
     * @param topic
     **/
    @GetMapping(value = "/topic-config", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
    public String topicConfig(@RequestParam String topic) {
        String result = JSON.toJSONString(KafkaUtils.topicConfig(topic));
        LOGGER.info(result);
        return result;
    }

    /**
     * 获取所有topic的配置
     **/
    @GetMapping(value = "/topics-configs", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
    public String topicsConfigs() {
        String result = JSON.toJSONString(KafkaUtils.topicsConfig());
        LOGGER.info(result);
        return result;
    }

    /**
     * 展示在某个分组中的consumer
     *
     * @param group
     **/
    @GetMapping(value = "/consumers-in-group", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
    public String consumersInGroup(@RequestParam String group) {
        String result = JSON.toJSONString(KafkaUtils.getConsumersInGroup(group));
        LOGGER.info(result);
        return result;
    }

    /**
     * 展示在某个分组中的consumer，按照topic划分
     *
     * @param group
     **/
    @GetMapping(value = "/consumers-per-topic", produces = MediaType.APPLICATION_JSON_UTF8_VALUE)
    public String consumersPerTopic(@RequestParam String group) {
        String result = JSON.toJSONString(KafkaUtils.getConsumersPerTopic(group));
        LOGGER.info(result);
        return result;
    }

    /**
     * 展示消费者消费详情
     *
     * @param topic
     * @param group
     **/
    @GetMapping("/consumer-detail")
    public String consumerDetail(@RequestParam String topic, @RequestParam String group) {
        try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
            //scala 输出流重定向
            Console.setOut(new PrintStream(bos));
            KafkaUtils.consumerDetail(topic, group);
            String result = bos.toString();
            LOGGER.info(result);
            return String.format("%s%s%s", "<pre>", result, "</pre>");
        } catch (Exception e) {
            LOGGER.error("获取消费详情", e);
        }
        return StringUtils.EMPTY;
    }

    /**
     * 消费消息并展示消息
     *
     * @param topic
     * @param group
     * @param size 消费消息数量
     * @param ack 消费的消息是否需要进行ack操作
     **/
    @GetMapping("/consumer-consume")
    public String consumerConsume(@RequestParam String topic
            , @RequestParam(required = false, defaultValue = "default") String group
            , @RequestParam(required = false, defaultValue = "1") int size
            , @RequestParam(required = false, defaultValue = "false") boolean ack) {
        List<String> messages;
        if (ack) {
            messages = ackConsumer.consume(topic, group, size);
        } else {
            messages = noAckConsumer.consume(topic, group, size);
        }
        return JSON.toJSONString(messages);
    }

    /**
     * 运行kafka相关命令
     *
     * @param sshRemote 连接远程主机命令（ssh user@host）
     * @param sshCommond kafka相关命令 (kafka-consumer.sh ...)
     **/
    @PostMapping("/commond-execute")
    public String commondExecute(
            @RequestParam(required = false) String sshRemote
            , @RequestParam String sshCommond
    ) {
        String commond = sshCommond + StringUtils.EMPTY;
        if (StringUtils.isNotBlank(sshRemote)) {
            commond = String.format("%s \"%s\"", sshRemote, commond);
        }
        String result = KafkaUtils.executeCommond(commond);
        return String.format("%s%s%s", "<pre>", result, "</pre>");
    }
}

四、消费配置

　　消费逻辑

package com.mochasoft.latte.data.kafka.consumer;

import java.util.Properties;
import kafka.consumer.ConsumerConfig;
import org.apache.commons.lang3.StringUtils;

public class KafkaConsumerConfig
{
  private String zkConnect;
  private String zkSessionTimeoutMs;
  private String zkSyncTimeMs;
  private String autoCommitIntervalMs;
  private String groupId = "default";
  
  static enum OffSet
  {
    smallest,  largest;
    
    private OffSet() {}
  }
  
  private OffSet offset = OffSet.largest;
  private Properties properties;
  
  public KafkaConsumerConfig()
  {
    this.properties = new Properties();
  }
  
  public KafkaConsumerConfig(String zkConnect, String zkSessionTimeoutMs, String zkSyncTimeMs, String autoCommitIntervalMs)
  {
    this.zkConnect = zkConnect;
    this.zkSessionTimeoutMs = zkSessionTimeoutMs;
    this.zkSyncTimeMs = zkSyncTimeMs;
    this.autoCommitIntervalMs = autoCommitIntervalMs;
    this.properties = new Properties();
  }
  
  public String getZkConnect()
  {
    return this.zkConnect;
  }
  
  public void setZkConnect(String zkConnect)
  {
    this.zkConnect = zkConnect;
  }
  
  public String getZkSessionTimeoutMs()
  {
    return this.zkSessionTimeoutMs;
  }
  
  public void setZkSessionTimeoutMs(String zkSessionTimeoutMs)
  {
    this.zkSessionTimeoutMs = zkSessionTimeoutMs;
  }
  
  public String getZkSyncTimeMs()
  {
    return this.zkSyncTimeMs;
  }
  
  public void setZkSyncTimeMs(String zkSyncTimeMs)
  {
    this.zkSyncTimeMs = zkSyncTimeMs;
  }
  
  public String getAutoCommitIntervalMs()
  {
    return this.autoCommitIntervalMs;
  }
  
  public void setAutoCommitIntervalMs(String autoCommitIntervalMs)
  {
    this.autoCommitIntervalMs = autoCommitIntervalMs;
  }
  
  public String getGroupId()
  {
    return this.groupId;
  }
  
  public void setGroupId(String groupId)
  {
    if (StringUtils.isNotBlank(groupId)) {
      this.groupId = groupId;
    }
  }
  
  public OffSet getOffset()
  {
    return this.offset;
  }
  
  public void setOffset(OffSet offset)
  {
    this.offset = offset;
  }
  
  public ConsumerConfig getConsumerConfig()
  {
    return new ConsumerConfig(getProperties());
  }
  
  public Properties getProperties()
  {
    if (StringUtils.isBlank(this.zkConnect)) {
      throw new IllegalArgumentException("Blank zkConnect");
    }
    if (StringUtils.isNotBlank(this.zkSessionTimeoutMs)) {
      this.properties.put("zookeeper.session.timeout.ms", this.zkSessionTimeoutMs);
    }
    if (StringUtils.isNotBlank(this.zkSyncTimeMs)) {
      this.properties.put("zookeeper.sync.time.ms", this.zkSyncTimeMs);
    }
    if (StringUtils.isNotBlank(this.autoCommitIntervalMs)) {
      this.properties.put("auto.commit.interval.ms", this.autoCommitIntervalMs);
    }
    if (StringUtils.isNotBlank(this.offset.name())) {
      this.properties.put("auto.offset.reset", this.offset.name());
    }
    this.properties.put("group.id", getGroupId());
    
    this.properties.put("zookeeper.connect", this.zkConnect);
    
    return this.properties;
  }
}

public static final class NoAckConsumer extends TheConsumer {
    public NoAckConsumer(KafkaConsumerConfig kafkaConsumerConfig) {
        super(kafkaConsumerConfig, false);
        this.consumerConfigProperties.setProperty("auto.commit.enable", "false");
    }
}

public static final class AckConsumer extends TheConsumer {

    public AckConsumer(KafkaConsumerConfig kafkaConsumerConfig) {
        super(kafkaConsumerConfig, true);
        this.consumerConfigProperties.setProperty("auto.commit.enable", "true");
    }
}

public static class TheConsumer {
    protected Properties consumerConfigProperties;
    private boolean ack;
    private StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
    private StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());

    public TheConsumer(KafkaConsumerConfig kafkaConsumerConfig, boolean ack) {
        this.ack = ack;
        this.consumerConfigProperties = new Properties();
        this.consumerConfigProperties.putAll(kafkaConsumerConfig.getProperties());
    }

    /**
     * @param topic 主题
     * @param group 分组
     * @param size  消费数量
     **/
    public List<String> consume(String topic, String group, int size) {
        if (StringUtils.isNotBlank(group)) {
            this.consumerConfigProperties.setProperty("group.id", group);
        }
        ConsumerConnector consumerConnector = null;
        try {
            consumerConnector = Consumer.createJavaConsumerConnector(new ConsumerConfig(this.consumerConfigProperties));
            Map<String, Integer> topics = new HashMap<>(1);
            topics.put(topic, 1);
            Map<String, List<KafkaStream<String, String>>> streams = consumerConnector.createMessageStreams(topics, keyDecoder, valueDecoder);
            if (!(CollectionUtils.isEmpty(streams) || CollectionUtils.isEmpty(streams.get(topic)))) {
                List<String> messages = new ArrayList<>();
                KafkaStream<String, String> messageStream = streams.get(topic).get(0);
                for (ConsumerIterator<String, String> it = messageStream.iterator(); it.hasNext(); ) {
                    MessageAndMetadata<String, String> messageAndMetadata = it.next();
                    messages.add(messageAndMetadata.message());
                    if (this.ack) {
                        consumerConnector.commitOffsets();
                    }
                    if (size <= messages.size()) {
                        break;
                    }
                }
                return messages;
            }
        } catch (Exception e) {
            LOGGER.error(String.format("%s ack consume has errors. topic=%s, group=%s, size=%d.", this.ack ? "" : "no", topic, group, size), e);
        } finally {
            if (consumerConnector != null) {
                consumerConnector.shutdown();
            }
        }
        return Collections.EMPTY_LIST;
    }
}

　　消费测试

public class KafkaTest extends BaseUnitTest {

    private static Logger LOGGER = LoggerFactory.getLogger(KafkaTest.class);

    @Value("${kafka.connection.zkconnect}")
    private String zkConnect;

    private static final AutoZkClient zkClient = new AutoZkClient("127.0.0.1:2181");
    private static final String TEST_TOPIC = "message-center-biz-expiration-reminder-topic";
    private static final String TEST_GROUP = "hjz-group";

    @Autowired
    private NoAckConsumer noAckConsumer;

    @Autowired
    private AckConsumer ackConsumer;

    @Autowired
    private KafkaProducer kafkaProducer;

    private CountDownLatch finishCountDownLatch = new CountDownLatch(20);

    @Test
    public void testNoAckConsume() throws InterruptedException {
        class ConsumeRun implements Callable<List<String>> {
            private TheConsumer consumer;
            private CountDownLatch countDownLatch;

            public ConsumeRun(TheConsumer consumer, CountDownLatch countDownLatch) {
                this.consumer = consumer;
                this.countDownLatch = countDownLatch;
            }


            @Override
            public List<String> call() {
                try {
                    this.countDownLatch.await();
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
                List<String> messages = consumer.consume(TEST_TOPIC, TEST_GROUP, 8);
                finishCountDownLatch.countDown();
                return messages;
            }
        }

        ExecutorService executorService = Executors.newFixedThreadPool(20);
        CountDownLatch countDownLatch = new CountDownLatch(1);

        List<Future<List<String>>> noAckConsumeFutures = new ArrayList<>(), ackConsumeFutures = new ArrayList<>();

        for (int i = 0; i < 10; ++i) {
            ConsumeRun consumeRun = new ConsumeRun(this.noAckConsumer, countDownLatch);
            noAckConsumeFutures.add(executorService.submit(consumeRun));
        }
        for (int i = 0; i < 10; ++i) {
            ConsumeRun consumeRun = new ConsumeRun(this.ackConsumer, countDownLatch);
            ackConsumeFutures.add(executorService.submit(consumeRun));
        }
        countDownLatch.countDown();
        finishCountDownLatch.await();

        System.out.println("no ack consumers response....");
        noAckConsumeFutures.forEach(future -> {
            try {
                System.out.println(future.get());
            } catch (Exception e){
            }
        });
        System.out.println("\n\nack consumers response....");
        ackConsumeFutures.forEach(future -> {
            try {
                System.out.println(future.get());
            } catch (Exception e) {
                e.printStackTrace();
            }
        });
    }

    @Test
    public void testProduce() {
        for (int i = 0; i < 100; ++i) {
            kafkaProducer.send(TEST_TOPIC, String.valueOf(i), "message " + i);
        }
        KafkaUtils.consumerDetail(TEST_TOPIC, TEST_GROUP);
    }

    public static void createTopic() {
        MessageCenterConstants.TopicConfig topicConfig = new MessageCenterConstants.TopicConfig();
        topicConfig.setName("kafka-test");
        KafkaMessageConfig.KafkaUtils.topicCreate(topicConfig);
    }

    public static void testKafka() {
        createTopic();
        System.out.println(KafkaUtils.topicsList());
        Properties properties = new Properties();
        properties.put("min.cleanable.dirty.ratio", "0.3");
        KafkaMessageConfig.KafkaUtils.topicChangeConfig(TEST_TOPIC, properties);
        System.out.println(KafkaMessageConfig.KafkaUtils.topicConfig(TEST_TOPIC));
        KafkaUtils.topicAlterPartitions(TEST_TOPIC, 7);
        KafkaMessageConfig.KafkaUtils.topicDescribe(TEST_TOPIC);
        kafka.utils.ZkUtils.getSortedBrokerList(zkClient);
    }

    public static void testTopicDescribe() {
        KafkaUtils.setZkClient(zkClient);
        new MessageCenterStateController().topicDescribe("message-center-recharge-transaction-push-topic");
    }

    public static void testConsumerDescribe() {
        KafkaUtils.setZkClient(zkClient);
        String[] args = {
                "--zookeeper", zkClient.getZkServers(),
                "--group", "",
                "--topic", "message-center-recharge-transaction-push-topic"
        };

        ConsumerOffsetChecker.main(args);
    }

    public static void testConsumerList() {
        KafkaUtils.setZkClient(zkClient);
        String[] args = {
                "--broker-list", zkClient.getZkServers(),
                "--topic", "message-center-recharge-transaction-push-topic",
                "--list"
        };

        SimpleConsumerShell.main(args);
    }

    public static void main(String[] args) {
        testConsumerList();
    }
}

　　测试no ack 以及 ack的消费结果

no ack consumers response....
[message 8, message 14, message 23, message 32, message 41, message 50, message 8, message 14]
[message 14, message 23, message 32, message 41, message 50, message 12, message 21, message 30]
[message 17, message 26, message 35, message 44, message 53, message 62, message 71, message 80]
[message 19, message 28, message 37, message 46, message 55, message 64, message 73, message 82]
[message 89, message 98, message 89, message 98, message 19, message 28, message 37, message 46]
[message 0, message 39, message 48, message 57, message 66, message 75, message 84, message 93]
[message 1, message 49, message 58, message 67, message 76, message 85, message 94, message 77]
[message 8, message 14, message 23, message 32, message 41, message 50, message 89, message 98]
[message 17, message 26, message 35, message 44, message 53, message 62, message 71, message 80]
[message 2, message 59, message 68, message 77, message 86, message 95, message 0, message 39]


ack consumers response....
[message 7, message 13, message 22, message 31, message 40, message 5, message 11, message 20]
[message 17, message 26, message 35, message 44, message 53, message 62, message 71, message 80]
[message 77, message 86, message 95, message 67, message 76, message 85, message 94, message 0]
[message 9, message 15, message 24, message 33, message 42, message 51, message 60, message 6]
[message 4, message 10, message 79, message 88, message 97, message 2, message 59, message 68]
[message 29, message 38, message 47, message 56, message 65, message 74, message 83, message 92]
[message 16, message 25, message 34, message 43, message 52, message 61, message 70, message 8]
[message 18, message 27, message 36, message 45, message 54, message 63, message 72, message 81]
[message 3, message 69, message 78, message 87, message 96, message 1, message 49, message 58]
[message 14, message 23, message 32, message 41, message 50, message 89, message 98, message 12]

　　消费测试结果分析：no ack的consumer可以实现消息的窥探。

 

五、效果图

　　获取topic详情

　　

　　获取所有的topic信息

　　

 　　获取消费详情

 　

 　　窥探消息

　　

 

转载于:https://www.cnblogs.com/hujunzheng/p/9327927.html

aipiannian6725
关注
  • 0
    点赞
  • 踩
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
Kafka工具类（Scala）
hyunbar的博客
08-27 826
1、配置文件config.properties # Kafka配置 kafka.broker.list = hadoop300:9092,hadoop301:9092,hadoop302:9092 # Redis配置 redis.host=hadoop300 redis.port=6379 2、读取Properties package com.duoduo.realtime.u...
kafkaUtils工具类封装
石榴姐yyds
03-09 2992
import java.util.Properties; import java.util.concurrent.Future; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.Producer; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka..
Java在线教育项目 第七天kafka实战应用&文章自动审核
fegus的博客
04-18 2001
kafka实战应用&文章自动审核 今日目标 熟悉kafka的封装技巧 熟悉阿里审核图片和文本内容审核 完成自媒体文章审核代码 完成自媒体端发布文章发送消息 完成admin端接收消息并自动审核 1 kafka封装 1.1 功能需求 消息对于现代软件项目来说，占有很重要的地位；同时市场上也发展处ActiveMq、RabbitMQ、Kafka、RocketMQ、Pulsar等众多优秀的框架；这些优秀的框架都由自身的特点和擅长的业务领域，在大数据领域中Kafka目前是使用较多的框架，Pulsar是一个后起之
「Kafka核心源码剖析系列」2. Kafka元数据以及消息封装
大数据与人工智能
08-23 451
作者 | 吴邪 大数据4年从业经验，目前就职于广州一家互联网公司，负责大数据基础平台自研、离线计算&实时计算研究编辑 | auroral-L全文共4296字，预计阅读35分钟。...
kafka java连接操作
常年被追砍的博客
05-16 3917
Java连接kafka操作，单机版kafka 代码记录如下 1、maven添加依赖配置： <dependency> <groupId>org.apache.kafka</groupId> <artifactId>kafka_2.10</artifactId> ...
kafka开发运维实战分享.pptx
04-28
- **0.10.x**：引入 Kafka Streams，消费者 API 稳定，生产者 API 性能提升。 - **0.11.x**：支持 Exactly-Once 语义，Producer 端的消息幂等性，以及事务特性。 - **1.x**：完善 Kafka Streams 和 Kafka Connect，...
Spark Streaming 流式处理整合Kafka.rar
最新发布
10-18
在示例代码中 `kafkaParams` 封装了 Kafka 消费者的属性，这些属性和 Spark Streaming 无关，是 Kafka 原生 API 中就有定义的。其中服务器地址、键序列化器和值序列化器是必选的，其他配置是可选的。 Spark ...
kafka-node-docker-mysql-example:dockerized节点，kafka和mysql API
05-15
kafka-node-docker-mysql-example dockerized节点，kafka和mysql，mongodb api卡夫卡使用node-rdkafka封装c ++ rdkafka库。 由暴雪维护运行和安装运行docker-compose build来构建npm镜像。 运行docker-compose up...
Spring boot集成Kafka消息中间件代码实例
08-19
* kafka-clients：用于提供Kafka客户端API * fastjson：用于JSON序列化和反序列化 二、配置文件 在application.properties文件中配置Kafka的代理地址、消费者组ID、自动提交offset等参数： * server.port=4400：...
phpkafkaconsumer是一个kafkaconsumer库支持group和rebalance
08-08
php-kafka-consumer 主要是对 php_rdkafka 的 consumer 的 API 进行一层封装，增加了原程序中所没有的与 zookeeper 交互的功能。
kafka封装源码
05-16
kafka java生产-消费调用代码 kafka java调用代码 kafka java调用代码
kafka之五 kafkaAdmin API
追赶者的博客
01-26 6550
kafka 创建topic javaAPI
KafkaAdminClient-概述
zc0565的博客
10-29 1525
在新版的 Kafka 中的 ZKUtils、AdminUtils 方法都已经过时了，目前 Kafka 提供了一个新的 API 工具 AdminClient。 官网对于 AdminClient 的介绍: The AdminClient API supports managing and inspecting topics, brokers, acls, and other Kafka object...
Kafka的集群管理工具AdminClient（替代了ZKUtils）
darkWatch的博客
02-16 6980
今天在使用的Kafka的时候，发现新版的Kafka中的ZKUtils方法显示已过时，查阅了资料发现ZKUtils方法已经被AdminClient替代了。 官网上关于AdminClient的介绍是：The AdminClient API supports managing and inspecting topics, brokers, acls, and other Kafka objects.，...
使用Java API进行kafka topic开发
meng984611383的博客
05-29 1万+
package org.sunny.two; import kafka.admin.AdminClient; import kafka.admin.AdminUtils; import kafka.admin.TopicCommand; import kafka.server.ConfigType; import kafka.utils.ZkUtils; import org.apache....
Kafka环境搭建以及服务封装
牛奋lch
11-18 4551
一、安装前准备 1、kafka版本：kafka_2.10-0.10.1.0.tgz 2、zookeeper版本：zookeeper-3.4.3.tar.gz 3、zookeeper集群： 192.168.1.108:2181,192.168.1.109:2181,192.168.1.110:2181 4、hosts文件中主机与ip映射关系 192.168.1.108 master 19
kafka API详解
热门推荐
.zz的博客
06-25 2万+
1. 生产者apiProducer是Kafka三大组件中的一个，用于发送消息到kafka集群中Producer提供了丰富的配置（见后面的配置项）用于控制它的行为在编码之前先使用命令创建topic./kafka-topics.sh --create --zookeeper hadoop01,hadoop02,hadoop03:2181 --partitions 2 --replication-fac...
Springboot Kafka生产者代码封装
07-16
在Spring Boot中封装Kafka生产者代码可以通过创建一个KafkaProducerService类来实现。 首先，需要在项目中引入Kafka客户端依赖，如前面所述。 然后，创建一个KafkaProducerService类，示例如下： ```java import ...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值