目录
1.3、 将topic mystreamin中的数据取出,再放入到topic streamout中
一、不经过任何数据处理,直接传输文件
1.1、初始化配置
Properties prop = new Properties();
prop.put(StreamsConfig.APPLICATION_ID_CONFIG, "mystream");
prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.91.180:9092");
prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
1.2、创建流构造器
StreamsBuilder builder = new StreamsBuilder();
1.3、 将topic mystreamin中的数据取出,再放入到topic streamout中
builder.stream("mystreamin").to("mystreamout");
上面一行相当于下面两行
KStream<Object, Object> mystreamin = builder.stream("mystreamin");
mystreamin.to("mystreamout");
1.4、通过建造者模式,创建kafkaStreams
Topology topo = builder.build();
KafkaStreams kafkaStreams = new KafkaStreams(topo, prop);
1.5启动程序
kafkaStreams.start();
二、传输过程中,进行一些数据处理
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import java.util.ArrayList;
import java.util.Properties;
public class UserFriendStream {
public static void main(String[] args) {
Properties properties = new Properties();
properties.put(StreamsConfig.APPLICATION_ID_CONFIG, "userfriend2");
properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.91.180:9092");
properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
properties.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1000);
//earliest latest one
properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
properties.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
properties.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
StreamsBuilder builder = new StreamsBuilder();
//主要业务逻辑处理 start
builder.stream("user_friends_raw")
.flatMap((key, value) -> {
ArrayList<KeyValue<String, String>> list = new ArrayList<>();
String[] fields = value.toString().split(",");
if (fields.length == 2) {
String userid = fields[0];
String[] friends = fields[1].split("\\s+");
for (String friendid : friends) {
System.out.println(userid + " " + friendid);
KeyValue<String, String> kv = new KeyValue<>(null, userid + "," + friendid);
list.add(kv);
}
}
return list;
})
.to("user_friends");
//主要业务逻辑处理 end
Topology topo = builder.build();
KafkaStreams streams = new KafkaStreams(topo, properties);
streams.start();
}
}