使用kafkaStream过滤数据

先写两个工具类

package my.test.test.events.stream;

import org.apache.kafka.streams.Topology;

/**
 * @author WGY
 */
public interface ICustomTopology {
    public Topology buildCustomTopology();



}

package my.test.test.events.stream;

import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;

import java.util.Properties;
import java.util.concurrent.CountDownLatch;

/**
 * @author WGY
 */
public class StreamHandler {
    private ICustomTopology topology;



    //构造方法
    public StreamHandler(ICustomTopology topology){
        this.topology=topology;
    }

    //执行器
    public void execute(){

        Properties prop = new Properties();
        prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop1:9092");
        prop.put(StreamsConfig.APPLICATION_ID_CONFIG,"kb071");
        prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,Serdes.String().getClass());


        Topology topo = this.topology.buildCustomTopology();

        KafkaStreams streams = new KafkaStreams(topo, prop);

        CountDownLatch countDownLatch = new CountDownLatch(1);
        Runtime.getRuntime().addShutdownHook(new Thread("kb07"){

            @Override
            public void run() {
                streams.close();
                countDownLatch.countDown();

            }
        });


        try {
            streams.start();
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }


}

写核心代码

package my.test.test.events.stream;

import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.kstream.KStream;

import java.util.ArrayList;
import java.util.List;

/**
 * @author WGY
 */
public class UserFriendsTopology implements ICustomTopology {
    @Override
    public Topology buildCustomTopology() {
        StreamsBuilder builder = new StreamsBuilder();
        //过滤数据
        final KStream<Object, Object> user_friends_raw = builder.stream("user_friends_raw")
                .filter((k, v) -> (!v.toString()
                        .startsWith("user,") && (v.toString())
                        .split(",")
                        .length == 2));

        user_friends_raw.flatMap((k,v)->{
            System.out.println(k+"       "+v);//1 ------2,3,4,5

            List<KeyValue<String,String>> keyValues = new ArrayList<>();

            String[] split = v.toString().split(",");//[1,2 3 5 6]
            String userID = split[0];
            String[] friends = split[1].split(" ");
            for (String friend:friends){
                KeyValue<String, String> keyValue = new KeyValue<>(null, userID+"  "+friend);
                keyValues.add(keyValue);
            }
            return keyValues;
        }).to("user_friends");

        Topology topo = builder.build();
        return topo;
    }
}

再写一个

package my.test.test.events.stream;

import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.kstream.KStream;

import java.util.ArrayList;
import java.util.List;

/**
 * @author WGY
 */
public class Event_attendee implements ICustomTopology{
    @Override
    public Topology buildCustomTopology() {
        StreamsBuilder builder = new StreamsBuilder();
        //过滤数据
        final KStream<Object, Object> user_friends_raw = builder.stream("event_attendees_raw")
                .filter((k, v) -> (!v.toString()
                        .startsWith("event,") && (v.toString())
                        .split(",")
                        .length == 5));
        user_friends_raw.flatMap((k,v)->{
            System.out.println(k+"       "+v);//1 ------2,3,4,5

            List<KeyValue<String,String>> keyValues = new ArrayList<>();

            String[] split = v.toString().split(",");//[1,2 3 5 6]
            String event = split[0];
            String[] yess = split[1].split(" ");
            String[] maybes = split[2].split(" ");
            String[] inviteds = split[3].split(" ");
            String[] nos = split[4].split(" ");
            for (String yes:yess){
                KeyValue<String, String> keyValue = new KeyValue<>(null, event+"--"+yes+"--yes");
                keyValues.add(keyValue);
            }
            for (String maybe:maybes){
                KeyValue<String, String> keyValue = new KeyValue<>(null, event+"--"+maybe+"--maybe");
                keyValues.add(keyValue);
            }
            for (String invited:inviteds){
                KeyValue<String, String> keyValue = new KeyValue<>(null, event+"--"+invited+"--invited");
                keyValues.add(keyValue);
            }
            for (String no:nos){
                KeyValue<String, String> keyValue = new KeyValue<>(null, event+"--"+no+"--no");
                keyValues.add(keyValue);
            }
            return keyValues;
        }).to("event_attendees");

        Topology topo = builder.build();
        return topo;

    }
}

测试运行

package my.test.test.events;

import my.test.test.events.stream.Event_attendee;
import my.test.test.events.stream.StreamHandler;
import my.test.test.events.stream.UserFriendsTopology;

/**
 * @author WGY
 */
public class StreamDriver {
    public static void main(String[] args) {
        Event_attendee event_attendee = new Event_attendee();
//        UserFriendsTopology topology = new UserFriendsTopology();
        StreamHandler handler = new StreamHandler(event_attendee);
        handler.execute();
    }
}