1、安装kafka(mac版):brew install kafka
2、启动zookeeper、kafka、flink:
zookeeper:/usr/local/Cellar/zookeeper/3.4.13/bin/zkServer start
kafka:/usr/local/Cellar/kafka/2.1.1/bin/kafka-server-start /usr/local/etc/kafka/server.properties
flink:/usr/local/Cellar/apache-flink/1.7.1/libexec/bin/start-cluster.sh (接着就可以进入web页面(http://localhost:8081/))
3、创建topic,名为tuzisir:
kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic tuzisir
4、新建flink(java版)项目:
mvn archetype:generate \
-DarchetypeGroupId=org.apache.flink \
-DarchetypeArtifactId=flink-quickstart-java \
-DarchetypeVersion=1.7.0 \
-DgroupId=net.qutoutiao.di \
-DartifactId=flink-logserver-java \
-Dversion=1.0 \
-Dpackage=net.qutoutiao.di \
-DinteractiveMode=false
并将以下依赖添加到pom.xml中
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_2.11</artifactId>
<version>1.7.2</version>
</dependency>
代码如下:
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import java.util.Properties;
public class ProducerKafkaFlink {
public static void main(String[] args) throws Exception {
final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
//从kafka读取数据
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
properties.setProperty("group.id", "test-consumer-group");
FlinkKafkaConsumer<String> myConsumer = new FlinkKafkaConsumer<>("tuzisir", new SimpleStringSchema(), properties);
DataStream<String> stream = env.addSource(myConsumer);
stream.print();
//将结果写到kafka
stream.addSink(new FlinkKafkaProducer<>(
"localhost:9092",
"student-write",
new SimpleStringSchema()
)).name("flink-connectors-kafka");
env.execute("write to kafka");
}
}