将表格数据从kafka提取出，编辑后再存入kafka。

一、功能需求

将表格去掉表头，分成map型数据

二、 代码

import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.kstream.KStream;

import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;

public class MyUserFriends {
    public static void main(String[] args) {
        Properties prop=new Properties();
        prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.255.11:9092");
        prop.put(StreamsConfig.APPLICATION_ID_CONFIG,"kb07");
        prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG,Serdes.String().getClass());
        prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,Serdes.String().getClass());

        StreamsBuilder builder=new StreamsBuilder();

        KStream<Object, Object> user_friends_raw = builder.stream("user_friends_raw")
                .filter((k, v) -> (!v.toString().startsWith("user,") && (v.toString().split(",").length == 2)));

        user_friends_raw.flatMap((k,v)->{
            System.out.println(k+"  "+v);
            List<KeyValue<String,String>> keyValues=new ArrayList<>();
            String[] split = v.toString().split(",");
            String userId=split[0];
            String[] friends = split[1].split("\\s");
            for (String friend:friends) {
                KeyValue<String,String> keyValue=new KeyValue<>(null,userId+" "+friend);
                keyValues.add(keyValue);
            }
            return keyValues;
        }).to("user_friends");

        Topology topo = builder.build();

        KafkaStreams streams = new KafkaStreams(topo, prop);

        CountDownLatch countDownLatch = new CountDownLatch(1);

        Runtime.getRuntime().addShutdownHook(new Thread("kb07"){
            @Override
            public void run() {
                streams.close();
                countDownLatch.countDown();
            }
        });

        streams.start();
        try {
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        System.exit(0);
    }
}

架构型代码

一、接口

package stream;

import org.apache.kafka.streams.Topology;

public interface ICustomTopology {
    public Topology buildCustomTopology();
}

二、实现类

package stream;

import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.kstream.KStream;

import java.util.ArrayList;
import java.util.List;

public class UserFriendsTopology implements ICustomTopology {
    StreamsBuilder builder=new StreamsBuilder();

    @Override
    public Topology buildCustomTopology() {

        KStream<Object, Object> user_friends_raw = builder.stream("user_friends_raw")
                .filter((k, v) -> (!v.toString().startsWith("user,") && (v.toString().split(",").length == 2)));
        user_friends_raw.flatMap((k,v)->{
            System.out.println(k+"  "+v);
            List<KeyValue<String,String>> keyValues=new ArrayList<>();
            String[] split = v.toString().split(",");
            String userId=split[0];
            String[] friends = split[1].split("\\s");
            for (String friend:friends) {
                KeyValue<String,String> keyValue=new KeyValue<>(null,userId+" "+friend);
                keyValues.add(keyValue);
            }
            return keyValues;
        }).to("user_friends");

        Topology topo = builder.build();
        return topo;
    }

}

三、stream类

package stream;

import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;

import java.util.Properties;
import java.util.concurrent.CountDownLatch;

public class StreamsHandler {

    private ICustomTopology topology;

    Properties prop = new Properties();

    public StreamsHandler(ICustomTopology topology){
        this.topology=topology;
    }

    public void execute(){
        prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.255.11:9092");
        prop.put(StreamsConfig.APPLICATION_ID_CONFIG,"kb07kkk");
        prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,Serdes.String().getClass());

        Topology topo = this.topology.buildCustomTopology();

        KafkaStreams streams = new KafkaStreams(topo, prop);

        CountDownLatch countDownLatch = new CountDownLatch(1);

        Runtime.getRuntime().addShutdownHook(new Thread("kb07-1"){
            @Override
            public void run() {
                streams.close();
                countDownLatch.countDown();
            }
        });

        streams.start();
        try {
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }


}

四、driver类

package stream;

public class streamsDriver {
    public static void main(String[] args) {
        ICustomTopology topology=new UserFriendsTopology();
        StreamsHandler handler=new StreamsHandler(topology);
        handler.execute();
    }
}