kafka连接异常

最新推荐文章于 2023-12-19 17:32:52 发布

「已注销」

最新推荐文章于 2023-12-19 17:32:52 发布

阅读量838

点赞数

文章标签：大数据 java python

原文链接：https://my.oschina.net/shyloveliyi/blog/900487

版权

2019独角兽企业重金招聘Python工程师标准>>>

kafka版本是0.10.2.1 本地java客户端版本是0.8.1.1

主要两个错误

第一个是连接拒绝

kafka Connection refused: no further information

然后发现第二个错误

Selector.poll(Selector.java:276) - Error in I/O with localhost/127.0.0.1

怀疑是ip绑定有问题，编辑server.properties，指定ip地址

advertised.host.name=ip地址

重启后，运行客户端，抛出另外一个问题

KafkaException: Failed to construct kafka producer

可能是因为客户端版本过低导致，升级客户端版本后，运行正常。

以下测试demo

生产者

public class ProducerDemo {

	private static final Logger LOG = Logger.getLogger(ProducerDemo.class);

	private static Properties properties = new Properties();

	static {
		properties.put("bootstrap.servers", "172.16.41.151:9092");
		properties.put("acks", "all");
		properties.put("retries", 0);
		properties.put("batch.size", 16384);
		properties.put("linger.ms", 1);
		properties.put("buffer.memory", 33554432);
		properties.put("key.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
		properties.put("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
	}

	public void produce() {
		KafkaProducer<byte[], byte[]> kafkaProducer = new KafkaProducer<byte[], byte[]>(properties);
		ProducerRecord<byte[], byte[]> kafkaRecord = new ProducerRecord<byte[], byte[]>("canal-t_canal",
				"kkk".getBytes(), "ssss".getBytes());
		kafkaProducer.send(kafkaRecord, new Callback() {
			public void onCompletion(RecordMetadata metadata, Exception e) {
				if (null != e) {
					LOG.info("the offset of the send record is {" + metadata.offset() + "}");
					LOG.error(e.getMessage(), e);
				}
				LOG.info("complete!");
			}
		});
		kafkaProducer.close();
	}

	public static void main(String[] args) {
		ProducerDemo kafkaProducerTest = new ProducerDemo();
		for (int i = 0; i < 10; i++) {
			kafkaProducerTest.produce();
		}
	}

}

消费者

public class ConsumerDemo {

	private static final String topic = "canal-t_canal";

	private static final Integer threads = 3;

	public static void main(String[] args) {

		Properties props = new Properties();

		props.put("zookeeper.connect", "172.16.41.151:2181");

		props.put("group.id", "1111");

		props.put("zookeeper.session.timeout.ms", "40000");
		props.put("zookeeper.sync.time.ms", "200");
		props.put("auto.commit.interval.ms", "1000");

		ConsumerConfig config = new ConsumerConfig(props);

		ConsumerConnector consumer = Consumer.createJavaConsumerConnector(config);

		Map<String, Integer> topicCountMap = new HashMap<String, Integer>();

		topicCountMap.put(topic, 1);

		Map<String, List<KafkaStream<byte[], byte[]>>> consumerMap = consumer.createMessageStreams(topicCountMap);

		List<KafkaStream<byte[], byte[]>> streams = consumerMap.get("canal-t_canal");

		for (final KafkaStream<byte[], byte[]> kafkaStream : streams) {

			new Thread(new Runnable() {

				public void run() {

					for (MessageAndMetadata<byte[], byte[]> mm : kafkaStream) {

						String msg = new String(mm.message());

						System.out.println(msg);

					}

				}

			}).start();

		}

	}

}

pom

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>scc-stu</groupId>
	<artifactId>stu-kafka</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<packaging>war</packaging>
	<name>stu-kafka</name>
	<description>stu-kafka</description>
	<dependencies>
		<dependency>
			<groupId>org.apache.kafka</groupId>
			<artifactId>kafka-clients</artifactId>
			<version>0.10.0.0</version>
		</dependency>

		<dependency>
			<groupId>org.apache.kafka</groupId>
			<artifactId>kafka_2.11</artifactId>
			<version>0.10.0.0</version>
		</dependency>
		<!-- log4j -->
		<dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>1.6.1</version>
        </dependency>
	</dependencies>
</project>

转载于:https://my.oschina.net/shyloveliyi/blog/900487