-
- 先建立好分区的topic
kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic flinkwrite5
建立分区为3 ,并行度为3 的producer
2.然后在flink中连接这个topic
treamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
Properties properties = new Properties();
properties.setProperty("bootstrap.servers", "localhost:9092");
DataStream<String> stream = env.addSource(new SimpleStringGenerator());
stream.addSink(new FlinkKafkaProducer010("flink_Producer", new SimpleStringSchema(), properties)).setParallelism(3);
env.execute()
通过从写SourceFunction来实现数据的生产
public static class StringInput implements SourceFunction<String> {
long i = 0;
boolean swith = true;
String s= "fngkjdgndfngdf";
@Override
public void run(SourceContext<String> ctx) throws Exception {
for(int k=0;k<5;k++) {
ctx.collect("flink:"+s.substring(k,k+2) +" "+ s.substring(k,k+2));
}
}
@Override
public void cancel() {
swith = false;
}
}
}
然后在新建一个类,用来消费这个producer
public class kafkaConsumer {
public static void main(String args[]) throws Exception {
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
env.setParallelism(3);
Properties properties = new Properties();
properties.setProperty("bootstrap.servers","localhost:9092");
properties.setProperty("group.id","consumer-group");
properties.setProperty("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
properties.setProperty("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
properties.setProperty("auto.offset.reset","latest");
FlinkKafkaConsumer010<String> myConsumer = new FlinkKafkaConsumer010<String>("flink_Producer", new SimpleStringSchema(), properties);
// myConsumer.setStartFromEarliest();
myConsumer.setStartFromLatest();
Map<KafkaTopicPartition, Long> specificStartOffsets = new HashMap<>();
specificStartOffsets.put(new KafkaTopicPartition("flink_Producer", 0), 23L);
specificStartOffsets.put(new KafkaTopicPartition("flink_Producer", 1), 31L);
specificStartOffsets.put(new KafkaTopicPartition("flink_Producer", 2), 43L);
myConsumer.setStartFromSpecificOffsets(specificStartOffsets);
DataStream<String> stream = env.addSource(myConsumer);
stream.flatMap(new FlatMapFunction<String, String>() {
@Override
public void flatMap(String value, Collector<String> out)
throws Exception {
for(String word: value.split(" ")){
out.collect(word);
}
}
});
Properties properties1 = new Properties();
properties1.setProperty("bootstrap.servers","localhost:9092");
stream.print();
System.out.println("teset");
env.execute("kafka sink");
}
}