Kafka的案例

最新推荐文章于 2024-07-27 11:12:23 发布

CesarChoy

最新推荐文章于 2024-07-27 11:12:23 发布

阅读量245

点赞数

分类专栏： Kafka

本文链接：https://blog.csdn.net/weixin_42687074/article/details/94554939

版权

Kafka 专栏收录该内容

3 篇文章 0 订阅

订阅专栏

前文：

一、客户端命令

二、Java的API

package cn.tedu.kafka;

import java.util.Arrays;
import java.util.Properties;
import java.util.concurrent.ExecutionException;

import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.junit.Test;

public class KafkaDemo {

	@Test
	public void producter() throws InterruptedException, ExecutionException {
		// 配置：key类型(一般为不自定义，不是主题)，value类型(一般为String类型)，集群服务列表
		Properties props = new Properties();
		props.put("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
		props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
		props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "hadoop01:9092,hadoop02:9092");

		// 生产者
		Producer<Integer, String> kafkaProducter = new KafkaProducer<>(props);
		for (int i = 0; i < 100; i++) {
			// 封装对象：主题名+数据
			ProducerRecord<Integer, String> message = new ProducerRecord<Integer, String>("enbook", "" + i);
			kafkaProducter.send(message);
		}
	}

	@Test
	public void comsumer_1() {
		// 集群服务列表、当前消费者线程所属消费组、消费数据的key类型、消费数据的value类型
		Properties props = new Properties();
		props.put("bootstrap.servers", "hadoop01:9092,hadoop02:9092");
		props.put("group.id", "comsumer-tutorial");
		props.put("key.deserializer", StringDeserializer.class.getName());
		props.put("value.deserializer", StringDeserializer.class.getName());

		// 消费者
		KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
		consumer.subscribe(Arrays.asList("enbook"));

		try {
			while (true) {
				// 封装对象，从分区的队列中获取数据(指定超时时间)
				ConsumerRecords<String, String> records = consumer.poll(Long.MAX_VALUE);
				for (ConsumerRecord<String, String> record : records) {
					System.out.println("偏移量:" + record.offset() + ",值：" + record.value() + ",分区号：" + record.partition());
				}
			}
		} catch (Exception e) {
		} finally {
			consumer.close();
		}
	}
}

三、Scala的API

package cn.tedu

import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.Seconds
import org.apache.spark.streaming.kafka.KafkaUtils

object kafka_test {
  def main(args: Array[String]): Unit = {
    //创建ssc
    val conf = new SparkConf();
    conf.setAppName("spark_kafka_test");
    conf.setMaster("local[5]");
    val sc = new SparkContext(conf);
    val ssc = new StreamingContext(sc, Seconds(3));

    //通过ssc获取和消费数据
    val stream = KafkaUtils.createStream(ssc, "hadoop01:2181,hadoop02:2181", "comsumer-tutorial", Map("enbook" -> 1))
    stream.map(_._2).print();

    //执行任务
    ssc.start();
    ssc.awaitTermination();
  }
}