【win10安装Kafka发消息及消费】

Zxdwr520

已于 2022-10-17 15:25:21 修改

阅读量1.1k

点赞数

分类专栏： SQL注入漏洞-Java 文章标签： java kafka spring spring cloud

于 2022-09-09 18:20:00 首次发布

本文链接：https://blog.csdn.net/Zxdwr520/article/details/126787292

版权

SQL注入漏洞-Java 专栏收录该内容

5 篇文章 0 订阅

订阅专栏

一、安装zookeepe的方式

1、zookeeper安装(Kafka2.8.0后不需要安装ZK)

下载安装文件：Index of /dist/zookeeper/zookeeper-3.4.10

Index of /dist/zookeeper/zookeeper-3.4.10https://archive.apache.org/dist/zookeeper/zookeeper-3.4.10/

解压后进入conf目录下zoo_sample.cfg重命名为zoo.cfg

进入zoo.cfg编辑

dataDir=D:\\zookeeper-3.4.10\\data

进入D:\zookeeper-3.4.10\bin件夹，以管理员身份运行zkServer.cmd、zkCli.cmd

2、安装kafka

下载安装包（下载二进制版本）：Apache Kafka

解压后编辑server.properties

log.dirs=D:\kafka_2.12-3.2.1\kafka-logs

log.retention.hours=168消息默认过期时间168h=7天

Kafka默认端口：9092，zookeeper默认端口：2181

启动kafka

.\bin\windows\kafka-server-start.bat .\config\server.properties

二、不需要安装zookeeper，只需安装Kafka

版本要>=2.8.0,博客实例用3.2.3版本的卡夫卡

下载kafka

https://kafka.apache.org/downloads

解压后

编辑如下文件

zookeeper.properties

dataDir=D:\kafka_2.12-3.2.3\zookeeper-logs

server.properties 修改

log.dirs=D:\kafka_2.12-3.2.3\kafka-logs

用cmd进入D:\kafka_2.12-3.2.3

启动zookeeper

bin\windows\zookeeper-server-start.bat config\zookeeper.properties

启动kafka

bin\windows\kafka-server-start.bat .\config\server.properties

引入依赖到pom.xml

 <!--kafka支持-->
        <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
        </dependency>

yml

server:
  port: 8089


#定时任务是否开启
scheduling:
  enable: true

#--------------------kafka相关配置start--------------------------------
spring:
  kafka:
    bootstrap-servers: localhost:9092 #集群方式：ip1:9092,ip2:9092,ip3:9092
    #设置一个默认组
    consumer:
      group-id: myGroup
      #是否开启自动提交
      enable-auto-commit: true
      #Kafka中没有初始偏移或如果当前偏移在服务器上不再存在时,默认区最新 ，有三个选项 【latest, earliest, none】
      auto-offset-reset: latest
      #key-value序列化反序列化
      key-deserializer: org.apache.kafka.common.serialization.StringDeserializer
      value-deserializer: org.apache.kafka.common.serialization.StringDeserializer
    producer:
      key-serializer: org.apache.kafka.common.serialization.StringSerializer
      value-serializer: org.apache.kafka.common.serialization.StringSerializer
      batch-size: 65536
      buffer-memory: 524288

#--------------------kafka相关配置end--------------------------------

控制器

@RestController
@Slf4j
public class KafkaController {
    
    @Autowired
    KafkaProducers kafkaProducers;

    @RequestMapping(value = "/kafka/test")
    public void run(){
        log.info("----------begin----------");
        //查询数据--这里仿接口造数据
        List<User> userList = new ArrayList<>();
        User user = new User();
        user.setId(1);
        user.setDateTime(new Date());
        user.setName("王八");
        user.setSex("男");
        user.setAge(20);
        userList.add(user);
        if(userList != null && userList.size() > 0) {
            try {
                ObjectMapper objectMapper = new ObjectMapper();
                String msg = objectMapper.writeValueAsString(userList);
                kafkaProducers.send(msg);
            } catch (Exception e) {
                e.printStackTrace();
                log.error(e.getMessage());
            }
        }
        log.info("----------end----------");
    }

}

kafka生产者

@Service
@Slf4j
public class KafkaProducers {
    
    public static final String topic = "myTopic";
    @Autowired
    KafkaTemplate<String, String> kafkaTemplate;

    public void send(String msg) {
        try {
            kafkaTemplate.send(topic,msg).addCallback(new ListenableFutureCallback() {
                @Override
                public void onFailure(Throwable throwable) {
                    log.error("There was an error sending the message.");
                }
                @Override
                public void onSuccess(Object o) {
                    log.info(" My message was sent successfully.");
                }
            });
        } catch (Exception e){
            e.printStackTrace();
        }
    }
}

消费者

@Component
@Slf4j
public class KafkaConsumer {
   
    @Autowired
    MyService myService;

    @KafkaListener(topics = "myTopic")
    public void runStart(ConsumerRecord<?, ?> record){
        ObjectMapper objectMapper = new ObjectMapper();
        Optional<?> kafkaMessage = Optional.ofNullable(record.value());
		List<Detail> objects = new ArrayList<>();
        try {
            if (kafkaMessage.isPresent()) {
                String message = (String) kafkaMessage.get();
                log.info("[runStart()-->]message: {}", message);
                List<User> userList = objectMapper.readValue(message, new TypeReference<List<User>>() {});
                if (!CollectionUtils.isEmpty(userList)) {
					for (User bean : userList) {
						Detail detail = new Detail();
						detail.setName(bean.getName);
						objects.add(detail);
					}
					try {
						if (objects.size() > 0) { //入库
							myService.saveBatch(objects);
						}
						} catch (Exception e) {
							e.printStackTrace();
						} 
				}
            }
        } catch (Exception e){
            e.printStackTrace();
            log.error(e.getMessage());
        }
    }
}

消费的主题： myTopic，默认0区

至此完成！！！

一下是另外的消费方式

@Slf4j
@Component
public class DispatchUserListener {
    

    @Resource
    MyService myService ;

    @KafkaListener(id = "dispatch-0",
        topicPartitions = {@TopicPartition(topic = "dispatch_bus", partitions = {"0"})})
    public void dispatchUserConsumer(String msg){
        ObjectMapper objectMapper = new ObjectMapper();
        DispatchBus dispatchBus = null;
        try {
            Map map = objectMapper.readValue(msg, Map.class);
            if ("4A".equals(map.get("package_type"))) {
                dispatchUser = objectMapper.readValue(msg, DispatchUser.class);
            }
        } catch (Exception e) { }

        if (dispatchUser == null ) {
            return;
        }
        myService .dispatch(dispatchUser);
    }

原生写法

/**
 * @Author: fan
 * @Description: 随便写写
 * @Date: 2022-09-03 15:16
 * @version: v1.0
 **/
@Component
@Slf4j
public class KafkaConsume {

    

    @Autowired
    IMyService myService;

    @Autowired
    IYouLogService youLogService;


    @Value("${kafka.topic-name.checkAdreal}")//主题可以放到配置文件中
    private String checkAdreal;

    public void runAdreal() {
        List<String> topicList = new ArrayList<>();
        topicList.add(checkAdreal);
        KafkaConfig kafkaConfig = SpringContextUtil.getBean("kafkaConfig", KafkaConfig.class);//KafkaConfig读kafka配置的信息，可以使用封装的kafkaTemplate都不用put
        Properties props = new Properties();
        props.put("bootstrap.servers", kafkaConfig.getZooKeeper());
        props.put("group.id", kafkaConfig.getGroupId());
        props.put("zookeeper.session.timeout.ms", kafkaConfig.getTimeout());
        props.put("zookeeper.sync.time.ms", kafkaConfig.getSyncTime());
        props.put("auto.commit.interval.ms", kafkaConfig.getInterval());

        props.put("auto.offset.reset", "latest");
        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        ObjectMapper objectMapper = new ObjectMapper();
        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        consumer.subscribe(topicList);
        Duration duration = Duration.ofMillis(100);

        Map<String,Object> map = new HashMap<>();
        List<UserVo> list = myService.getListAll(map);
        log.info("[KafkaConsume--runAdreal()] size={},topic={},bootstrap.servers={},group.id={}",
                list.size(),
                checkAdreal,
                kafkaConfig.getZooKeeper(),
                kafkaConfig.getGroupId());
        Integer i = 0;
        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(duration);
            for (ConsumerRecord<String, String> record : records) {
                try {
                    if (StringUtil.isNotEmpty(record.value())) {
                        Map<String, Object> mapKafka = objectMapper.readValue(record.value(), Map.class);
                        String order_id = (String) mapKafka.get("order_id");
                        String product_id = (String) mapKafka.get("product_id");
                        String order_number = (String) mapKafka.get("order_number");
                        if (CollectionUtils.isNotEmpty(list)
                                && StringUtil.isNotEmpty(order_id)
                                && StringUtil.isNotEmpty(product_id)
                                && StringUtil.isNotEmpty(order_number) ) {
                            for (UserVo user : list) {
                                Integer number = user.getNumber();
                                Integer orderId = user.getOrderId();
                                Integer productId = user.getProductId();

                                List<Object> idList = new ArrayList<>(Collections.singleton(mapKafka.get("id")));
                                map.put("paramter0",idList.toArray(new String[idList.size()]));
                                map.put("paramter1",user.getName());
                                map.put("paramter2",user.getAge());
                                map.put("paramter3",user.getSex());
                                map.put("paramter4",user.getScores());
                                map.put("paramter5",number);
                                map.put("paramter6",orderId);
                                map.put("paramter7",productId);
								map.put("paramter8",mapKafka.get("status"));
                                MyUtil.post(map);

                                YouLog youLog = new YouLog();
                                youLog.setid((Integer) mapKafka.get("id"));
                                youLog.setName((String) mapKafka.get("name"));
                                youLog.setContent( "你干了什么事：" );
                                youLog.setSendTime(new Date());
                                youLog.setSendBy("sys");
                                i += youLogService.save(youLog);
                            }
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    log.error("下发日志失败 error={}", e.getMessage());
                }finally {
                    log.info("[KafkaConsume--runAdreal()] size={}, topic={}, bootstrap.servers={}, group.id={}, i={}",
                            list.size(),
                            checkAdreal,
                            kafkaConfig.getZooKeeper(),
                            kafkaConfig.getGroupId(),
                            i);
                }
            }
        }
    }

}