MQTT+Flink实现实时消息的订阅与发布

一、添加MQTT依赖包

<dependency>
	<groupId>org.eclipse.paho</groupId>
	<artifactId>org.eclipse.paho.client.mqttv3</artifactId>
	<version>1.2.0</version>
</dependency>

二、Mqtt的配置类MqttConfig.java

import java.io.Serializable;
//该类需要实现序列化所以必须实现Serializable接口
public class MqttConfig implements Serializable {
 
    public MqttConfig(String username, String password, String hostUrl, String clientId, String msgTopic) {
        this.username = username;
        this.password = password;
        this.hostUrl = hostUrl;
        this.clientId = clientId;
        this.msgTopic = msgTopic;
    }
    //连接名称
    private String username;
    //连接密码
    private String password;
    //ip地址以及端口号
    private String hostUrl;
    //服务器ID注意不能与其他连接重复，否则会连接失败
    private String clientId;
    //订阅的主题
    private String msgTopic;
 
    //获得用户名
    public String getUsername() {
        return username;
    }
    //获得密码
    public String getPassword() {
        return password;
    }
    //获得客户端id
    public String getClientId() {
        return clientId;
    }
    //获得服务端url
    public String getHostUrl() {
        return hostUrl;
    }
    //获得订阅
    public String[] getMsgTopic() {
        String[] topic = msgTopic.split(",");
        return topic;
    }
}

三、Mqtt的实现类MqttConsumer.java

import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
import org.eclipse.paho.client.mqttv3.*;
import org.eclipse.paho.client.mqttv3.persist.MemoryPersistence;
import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
 
 
/**
 * MQTT客户端订阅消息类
 * @author zhongyulin
 *
 */
 
public class MqttConsumer extends RichParallelSourceFunction<String>{
    //存储服务
    private static MqttClient client;
    //存储订阅主题
    private static MqttTopic mqttTopic;
    //阻塞队列存储订阅的消息
    private BlockingQueue<String> queue = new ArrayBlockingQueue<String>(10);
 
    //包装连接的方法
    private void connect() throws MqttException {
        //配置连接参数
        MqttConfig mqttConfigBean = new MqttConfig("用户名", "密码", "tcp://服务器ip:1883", "mqtt", "自定义");
        //连接mqtt服务器
        client = new MqttClient(mqttConfigBean.getHostUrl(), mqttConfigBean.getClientId(), new MemoryPersistence());
        MqttConnectOptions options = new MqttConnectOptions();
        options.setCleanSession(false);
        options.setUserName(mqttConfigBean.getUsername());
        options.setPassword(mqttConfigBean.getPassword().toCharArray());
        options.setCleanSession(false);   //是否清除session
        // 设置超时时间
        options.setConnectionTimeout(30);
        // 设置会话心跳时间
        options.setKeepAliveInterval(20);
        try {
            String[] msgtopic = mqttConfigBean.getMsgTopic();
            //订阅消息
            int[] qos = new int[msgtopic.length];
            for (int i = 0; i < msgtopic.length; i++) {
                qos[i] = 0;
            }
            client.setCallback(new MsgCallback(client, options, msgtopic, qos){});
            client.connect(options);
            client.subscribe(msgtopic, qos);
            System.out.println("MQTT连接成功:" + mqttConfigBean.getClientId() + ":" + client);
        } catch (Exception e) {
            System.out.println("MQTT连接异常：" + e);
        }
    }
    //实现MqttCallback，内部函数可回调
    class MsgCallback implements MqttCallback{
        private MqttClient client;
        private MqttConnectOptions options;
        private String[] topic;
        private int[] qos;
 
        public MsgCallback() {
        }
 
        public MsgCallback(MqttClient client, MqttConnectOptions options, String[] topic, int[] qos) {
            this.client = client;
            this.options = options;
            this.topic = topic;
            this.qos = qos;
        }
        //连接失败回调该函数
        @Override
        public void connectionLost(Throwable throwable) {
            System.out.println("MQTT连接断开，发起重连");
            while (true) {
                try {
                    Thread.sleep(1000);
                    client.connect(options);
                    //订阅消息
                    client.subscribe(topic, qos);
                    System.out.println("MQTT重新连接成功:" + client);
                    break;
                } catch (Exception e) {
                    e.printStackTrace();
                    continue;
                }
            }
        }
        //收到消息回调该函数
        @Override
        public void messageArrived(String s, MqttMessage message) throws Exception {
            System.out.println();
            //订阅消息字符
            String msg = new String(message.getPayload());
            byte[] bymsg = getBytesFromObject(msg);
            System.out.println("topic:" + topic);
            queue.put(msg);
 
        }
        //对象转化为字节码
        public byte[] getBytesFromObject(Serializable obj) throws Exception {
            if (obj == null) {
                return null;
            }
            ByteArrayOutputStream bo = new ByteArrayOutputStream();
            ObjectOutputStream oo = new ObjectOutputStream(bo);
            oo.writeObject(obj);
            return bo.toByteArray();
        }
 
        @Override
        public void deliveryComplete(IMqttDeliveryToken iMqttDeliveryToken) {
 
        }
    }
    //flink线程启动函数
    @Override
    public void run(final SourceContext<String> ctx) throws Exception {
        connect();
        //利用死循环使得程序一直监控主题是否有新消息
        while (true){
            //使用阻塞队列的好处是队列空的时候程序会一直阻塞到这里不会浪费CPU资源
            ctx.collect(queue.take());
        }
    }
 
    @Override
    public void cancel() {
 
    }
 
    /**
     * 订阅某个主题
     *
     * @param topic
     * @param qos
     */
    public void subscribe(String topic, int qos) {
        try {
            System.out.println("topic:" + topic);
            client.subscribe(topic, qos);
        } catch (MqttException e) {
            e.printStackTrace();
        }
    }
 
    public MqttClient getClient() {
        return client;
    }
 
    public void setClient(MqttClient client) {
        this.client = client;
    }
 
    public MqttTopic getMqttTopic() {
        return mqttTopic;
    }
 
    public void setMqttTopic(MqttTopic mqttTopic) {
        this.mqttTopic = mqttTopic;
    }
 }

四、Flink工作启动类

import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
 
import java.util.*;
 
public class StreamingJob {
 
 
    /**
     * Skeleton for a Flink Streaming Job.
     *
     * <p>For a tutorial how to write a Flink streaming application, check the
     * tutorials and examples on the <a href="http://flink.apache.org/docs/stable/">Flink Website</a>.
     *
     * <p>To package your appliation into a JAR file for execution, run
     * 'mvn clean package' on the command line.
     *
     * <p>If you change the name of the main class (with the public static void main(String[] args))
     * method, change the respective entry in the POM.xml file (simply search for 'mainClass').
     */
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        //接收消息并将数据转化为流
        DataStream<String> stream = env.addSource(new MqttConsumer());
        //通过函数转化流的形式
        DataStream<Tuple2<String, String>> dataStream = stream.flatMap(new FlatMapFunction<String, Tuple2<String, String>>() {
            @Override
            public void flatMap(String value, Collector<Tuple2<String, String>> out) throws Exception {
                out.collect(Tuple2.of(str[0], str[1]));
            }
        }).setParallelism(1);//线程并行数量
        dataStream.print();
        env.execute();
    }
}