Apache Kafka;kafka下载地址,一定不要下源码,不要下错,要下Binary downloads
kafka3.0之后内置zk所以不需要额外下载
第一步:配置文件修改
server.properties
# broker.id 要保持唯一,只启动一台的话则不需修改
# The id of the broker. This must be set to a unique integer for each broker.
broker.id=0
# 修改日志存放位置
# A comma separated list of directories under which to store log files
log.dirs=D://kafka//kafka_2.13-3.1.0//kafka-logs
# 修改zk连接地址,默认本机启动则不需要修改
# Zookeeper connection string (see zookeeper docs for details).
# This is a comma separated host:port pairs, each corresponding to a zk
# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
# You can also append an optional chroot string to the urls to specify the
# root directory for all kafka znodes.
zookeeper.connect=localhost:2181
如果出现问题,还要添加一个参数:hostname:服务器的ip,(必须用IP连接)
zookeeper.properties
# 修改日志存放位置
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# the directory where the snapshot is stored.
dataDir=D://kafka//kafka_2.13-3.1.0//zookeeper-logs
第二步:启动
要先启动zk
启动语句
.\bin\windows\zookeeper-server-start.bat .\config\zookeeper.properties
然后启动kafka
启动语句
.\bin\windows\kafka-server-start.bat .\config\server.properties
进入/bin/windows 目录下通过下面的命令创建topic
kafka-topics.bat --bootstrap-server localhost:9092 --create --replication-factor 3 --partitions 1 --topic test
查看当前有哪些topic
kafka-topics.bat --bootstrap-server localhost:9092 --list
查看某个topic的详细信息
kafka-topics.bat --bootstrap-server localhost:9092 --describe --topic test
修改topic的配置
kafka-topics.bat --bootstrap-server localhost:9092 --alter --topic test --partitions 3
删除topic会报错,此时停掉所有的cmd,然后将日志文件删除,重新启动可以解决
启动生产者命令/bin/window 目录下
kafka-console-producer.bat --broker-list localhost:9092 --topic test
启动消费者命令/bin/window 目录下
kafka-console-consumer.bat --bootstrap-server localhost:9092 --topic test --from-beginning
然后就可以快乐的发送啦!
Java代码:
maven添加配置
<!--kafka依赖配置 *****开始******-->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.12</artifactId>
<version>2.1.1</version>
<scope>provided</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.6.3</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-streams</artifactId>
<version>1.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
</dependency>
<!--kafka依赖配置 *****结束******-->
生产者代码:
import java.util.Properties;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.serialization.StringSerializer;
public class KafkaProducerTest implements Runnable {
private final KafkaProducer<String, String> producer;
private final String topic;
private String clientid;
public KafkaProducerTest(String topicName,String clientid) {
Properties props = new Properties();
props.put("bootstrap.servers", "192.168.0.108:9092");
props.put("acks", "all");
props.put("retries", 0);
props.put("batch.size", 16384);
props.put("key.serializer", StringSerializer.class.getName());
props.put("value.serializer", StringSerializer.class.getName());
this.producer = new KafkaProducer<String, String>(props);
this.topic = topicName;
this.clientid = clientid;
}
@Override
public void run() {
int messageNo = 1;
try {
for(;;) {
String messageStr= "你好,这是第"+messageNo+"条数据 clientid=" + clientid;
producer.send(new ProducerRecord<String, String>(topic, "Message", messageStr));
//生产了100条就打印
if(messageNo%100==0){
System.out.println("发送的信息:" + messageStr);
}
//生产1000条就退出
if(messageNo == 1000){
System.out.println("成功发送了"+messageNo+"条");
break;
}
messageNo++;
}
} catch (Exception e) {
e.printStackTrace();
} finally {
producer.close();
}
}
public static void main(String args[]) {
KafkaProducerTest test1 = new KafkaProducerTest("test", "clientid1");
Thread thread1 = new Thread(test1);
thread1.start();
}
}
消费者代码:
import java.util.Arrays;
import java.util.Properties;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
public class KafkaConsumerTest implements Runnable {
private final KafkaConsumer<String, String> consumer;
private ConsumerRecords<String, String> msgList;
private final String topic;
private String clientid;
private static final String GROUPID = "groupA";
public KafkaConsumerTest(String topicName,String clientid) {
Properties props = new Properties();
props.put("bootstrap.servers", "192.168.0.108:9092");
props.put("group.id", GROUPID);
props.put("enable.auto.commit", "true");
props.put("auto.commit.interval.ms", "1000");
props.put("session.timeout.ms", "30000");
props.put("auto.offset.reset", "earliest");
props.put("key.deserializer", StringDeserializer.class.getName());
props.put("value.deserializer", StringDeserializer.class.getName());
this.consumer = new KafkaConsumer<String, String>(props);
this.topic = topicName;
this.consumer.subscribe(Arrays.asList(topic));
this.clientid = clientid;
}
@Override
public void run() {
int messageNo = 1;
System.out.println("---------开始消费---------");
try {
for (;;) {
msgList = consumer.poll(1000);
if(null!=msgList&&msgList.count()>0){
for (ConsumerRecord<String, String> record : msgList) {
//消费100条就打印 ,但打印的数据不一定是这个规律的
if(messageNo%100==0){
System.out.println(messageNo+"=======成功消费:receive: key = " + record.key() + ", value = " + record.value()+" offset==="+record.offset());
}
//当消费了1000条就退出
if(messageNo == 1000){
break;
}
messageNo++;
}
}else{
Thread.sleep(1000);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
consumer.close();
}
}
public static void main(String args[]) {
KafkaConsumerTest test1 = new KafkaConsumerTest("test", "clientid1");
Thread thread1 = new Thread(test1);
thread1.start();
}
}
自测没问题!