Kafka Streaming处理数据

用kafka streaming从kafka中拿数据，处理完数据后，再存到kafka中
有两种方式
第一种：直接全部写在main方法中

package com.wang.events;

import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.*;
import org.apache.kafka.streams.kstream.KStream;

import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;

public class MyUserFriends {
    public static void main(String[] args) {
        Properties prop = new Properties();
        prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop1:9092");
        prop.put(StreamsConfig.APPLICATION_ID_CONFIG,"kb07");
        prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG,Serdes.String().getClass());
        prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,Serdes.String().getClass());

        StreamsBuilder builder = new StreamsBuilder();

		//user_friends_raw是kafka中存放数据的topic，从中读取数据
        KStream<Object, Object> user_friends_raw = builder.stream("user_friends_raw")
                .filter((k, v) -> (!v.toString().startsWith("user,") && v.toString().split(",").length == 2));
         //传进来的(k,v)键值对，key没有数据，value是topic中的一行数据
        user_friends_raw.flatMap((k,v)->{
            System.out.println(k+" "+v);
            List<KeyValue<String,String>> keyValues = new ArrayList<>();
            //对v做操作
            String[] split = v.toString().split(",");
            String userId = split[0];
            String[] friends = split[1].split(" ");
            for (String friend : friends) {
                KeyValue<String,String> keyValue = new KeyValue<>(null,userId+" "+friend);
                keyValues.add(keyValue);
            }
            return keyValues;
            //to()写入的topic
        }).to("user_friends");

		//创建topo结构
        Topology topo = builder.build();
        //创建kafkastream，传入topo和prop
        KafkaStreams streams = new KafkaStreams(topo, prop);
		
		//线程计数，结束一个减一个，参数是线程运行数量
        CountDownLatch countDownLatch = new CountDownLatch(1);
        //如果所有都结束，就脱钩
        Runtime.getRuntime().addShutdownHook(new Thread("kb07"){
            @Override
            public void run() {
                streams.close();
                countDownLatch.countDown();
            }
        });


        try {
            streams.start();
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        System.exit(0);
    }
}

方式二：其实只需要注重开发 topo就可以，配置和运行都只需要写一次，可以将配置和运行封装起来
首先写一个ICustomTopology接口，里面写一个返回Topology类型的方法

package com.wang.stream;

import org.apache.kafka.streams.Topology;

public interface ICustomTopology {
    public Topology buildCustomTopology();
}

再写一个StreamHandler封装配置和运行

package com.wang.stream;

import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.StreamsConfig;
import org.apache.kafka.streams.Topology;

import java.util.Properties;
import java.util.concurrent.CountDownLatch;

public class StreamHandler {
	//定义一个ICustomTopology 类型，构造方法传入
    private ICustomTopology topology;
	
	//构造方法，用来传入ICustomTopology对象
    public StreamHandler(ICustomTopology topology) {
        this.topology = topology;
    }

    public void execute() {
        Properties prop = new Properties();
        prop.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop1:9092");
        prop.put(StreamsConfig.APPLICATION_ID_CONFIG,"kb073");
        prop.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG,Serdes.String().getClass());
        prop.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG,Serdes.String().getClass());

		//用传进来的ICustomTopology topology对象调用buildCustomTopology方法，返回Topology 
        Topology topo = this.topology.buildCustomTopology();
        KafkaStreams streams = new KafkaStreams(topo, prop);

        CountDownLatch countDownLatch = new CountDownLatch(1);
        //脱钩
        Runtime.getRuntime().addShutdownHook(new Thread("kb073"){
            @Override
            public void run() {
                streams.close();
                countDownLatch.countDown();
            }
        });


        try {
            streams.start();
            countDownLatch.await();
        } catch (InterruptedException e) {
            e.printStackTrace();
        }

        System.exit(0);
    }
}

程序员只需要专注写UserFriendsToplogy就行，实现ICustomTopology 接口，重写buildCustomTopology方法，返回Topology

package com.wang.stream;

import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.kstream.KStream;

import java.util.ArrayList;
import java.util.List;

public class UserFriendsToplogy implements ICustomTopology {
    StreamsBuilder builder = new StreamsBuilder();
    @Override
    public Topology buildCustomTopology() {

        final KStream<Object, Object> user_friends_raw = builder.stream("user_friends_raw")
                .filter((k, v) -> (!v.toString().startsWith("user,") && v.toString().split(",").length == 2));
        user_friends_raw.flatMap((k,v)->{
            System.out.println(k+" "+v);
            List<KeyValue<String,String>> keyValues = new ArrayList<>();
            String[] split = v.toString().split(",");
            String userId = split[0];
            String[] friends = split[1].split(" ");
            for (String friend : friends) {
                KeyValue<String,String> keyValue = new KeyValue<>(null,userId+" "+friend);
                keyValues.add(keyValue);
            }
            return keyValues;
        }).to("user_friends222");
        Topology topo = builder.build();
        return topo;
    }
}

主方法

package com.wang.stream;

public class StreamDriver {
    public static void main(String[] args) {
        ICustomTopology topology = new UserFriendsToplogy();
        StreamHandler handler = new StreamHandler(topology);
        handler.execute();
    }
}