Flink - RabbitMQ 简单使用

maven依赖

<?xml version="1.0" encoding="UTF-8"?>

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>com.hydrogen</groupId>
  <artifactId>flink_rabbitmq</artifactId>
  <version>1.0.0</version>
  <name>flink_rabbitmq</name>
  <url>http://www.example.com</url>

  <properties>
    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    <maven.compiler.source>1.8</maven.compiler.source>
    <maven.compiler.target>1.8</maven.compiler.target>
    <java.version>1.8</java.version>
    <flink.version>1.10.1</flink.version>
    <scala.version>2.11</scala.version>
  </properties>

  <dependencies>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.12</version>
    </dependency>
    
    <!-- flink-scala -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-scala_${scala.version}</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!-- flink-scala-streaming -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-streaming-scala_${scala.version}</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!-- rabbitMQ -->
    <dependency>
      <groupId>org.apache.flink</groupId>
      <artifactId>flink-connector-rabbitmq_${scala.version}</artifactId>
      <version>${flink.version}</version>
    </dependency>
    <!--fastJson-->
    <dependency>
      <groupId>com.alibaba</groupId>
      <artifactId>fastjson</artifactId>
      <version>1.2.72</version>
    </dependency>
    <!-- lombok -->
    <dependency>
      <groupId>org.projectlombok</groupId>
      <artifactId>lombok</artifactId>
      <version>1.18.10</version>
      <scope>provided</scope>
    </dependency>

  </dependencies>

  <build></build>
</project>

RabbitMQ数据写入工具类

package com.study.mq.util;

import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;

import java.io.IOException;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Scanner;
import java.util.concurrent.TimeoutException;

/**
 * @Date: 2021/3/2 下午 1:34
 * @Author: 冯照龙
 * @Description:
 * @URI:
 * @Version: 1.0
 */
public class RabbitMQUtil {

    public static void main(String[] args) throws IOException, TimeoutException {
        //连接配置
        String queueName = "test_01";
        Connection connection = getConnection();
        Channel channel = createChannel(connection,queueName);
        //读取控制台输入
        Scanner scanner = new Scanner(System.in);
        DateTimeFormatter pattern = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
        while (scanner.hasNext()){
            String msg = scanner.nextLine();
            System.out.println(LocalDateTime.now().format(pattern) + " 发送消息：'" + msg + "'");
            sendMessage(channel,queueName,msg);
        }
        //释放资源
        channel.close();
        connection.close();
    }

    /**
     * 建立连接
     * @return
     * @throws IOException
     * @throws TimeoutException
     */
    private static Connection getConnection() throws IOException, TimeoutException {
        ConnectionFactory factory = new ConnectionFactory();
        factory.setHost("flink1");
        factory.setPort(5672);
        //账号密码
        factory.setUsername("fzl");
        factory.setPassword("123");
        //虚拟主机
        factory.setVirtualHost("flink");
        return factory.newConnection();
    }

    /**
     * 创建通道
     * @param conn
     * @return
     * @throws IOException
     */
    private static Channel createChannel(Connection conn, String queueName) throws IOException {
        Channel channel = conn.createChannel();
        channel.queueDeclare(queueName,true,false,false,null);
        return channel;
    }

    /**
     * 写入数据
     * @param channel
     * @param queueName
     * @param message
     * @throws IOException
     */
    private static void sendMessage(Channel channel, String queueName, String message) throws IOException {
        channel.basicPublish("", queueName, null, message.getBytes("UTF-8"));
    }

}

读取test_01 写入test_02Job

从test_01读取，写入test_02

package com.study.mq

import java.time.LocalDateTime

import com.study.mq.sink.RabbitMQSink
import org.apache.flink.api.common.serialization.{SerializationSchema, SimpleStringSchema}
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.rabbitmq.{RMQSink, RMQSource}
import org.apache.flink.streaming.connectors.rabbitmq.common.RMQConnectionConfig

/**
 * @Date: 2021/3/2 下午 1:16
 * @Author: 冯照龙
 * @Description: 
 * @Version: 1.0
*/
object ReadRabbitMQ {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)

    //连接配置
    val config = new RMQConnectionConfig.Builder()
      .setHost("flink1")
      .setPort(5672)
      .setVirtualHost("flink")

      .setUserName("flink")
      .setPassword("123")
      .build()


    env
      .addSource(new RMQSource[String](config,"test_01",true, new SimpleStringSchema()))
//      .addSink(new RMQSink[String](config,"test_02",new SimpleStringSchema()))
      .addSink(new RabbitMQSink[String](config,"test_02",new SimpleStringSchema()))
//      .print()

    env.execute("test read mq")
  }
}

RMQSink报错

Caused by: com.rabbitmq.client.ShutdownSignalException: 
channel error; protocol method: #method<channel.close> 
(reply-code=406, reply-text=PRECONDITION_FAILED - inequivalent arg 'durable' for queue 'test_02' in vhost 'flink': 
received 'true' but current is 'false', class-id=50, method-id=10)

需要继承RMQSink，修改test_02持久化设置

继承RMQSink 重写setupQueue()

package com.study.mq.sink;

import org.apache.flink.api.common.serialization.SerializationSchema;
import org.apache.flink.streaming.connectors.rabbitmq.RMQSink;
import org.apache.flink.streaming.connectors.rabbitmq.common.RMQConnectionConfig;

import java.io.IOException;

/**
 * @Date: 2021/3/4 下午 2:01
 * @Author: 冯照龙
 * @Description:
 * @Version: 1.0
 */
public class RabbitMQSink<IN> extends RMQSink<IN> {

    /**
     * 重写构造方法
     * @param rmqConnectionConfig
     * @param queueName
     * @param schema
     */
    public RabbitMQSink(RMQConnectionConfig rmqConnectionConfig, String queueName, SerializationSchema<IN> schema) {
        super(rmqConnectionConfig, queueName, schema);
    }

    /**
     * 重写声明队列方法(开启持久化)
     * @throws IOException
     */
    @Override
    protected void setupQueue() throws IOException {
        if (super.queueName != null) {
            super.channel.queueDeclare(super.queueName,true,false,false,null);
        }
    }
}

读取test_02Job

package com.study.mq

import java.time.LocalDateTime
import java.time.format.DateTimeFormatter

import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.connectors.rabbitmq.RMQSource
import org.apache.flink.streaming.connectors.rabbitmq.common.RMQConnectionConfig

/**
 * @Date: 2021/3/2 下午 12:42
 * @Author: 冯照龙
 * @Description:
 * @Version: 1.0
*/
object FlinkJobRabbitMQ {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment

    env.setParallelism(1)

    val conf = new RMQConnectionConfig.Builder()
      .setHost("flink1")
      .setPort(5672)
      .setVirtualHost("flink")

      .setUserName("fzl")
      .setPassword("123")
      .build

    env
      .addSource(new RMQSource[String](conf,"test_02",true, new SimpleStringSchema()))
      .addSink(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")) + " => " + _)
//      .print()


    env.execute("test mq")
  }
}

测试流程