Flink 自定义sink 写入 RabbitMQ

添加依赖

<!-- rabbitmq -->
<dependency>
	<groupId>org.apache.flink</groupId>
	<artifactId>flink-connector-rabbitmq_2.12</artifactId>
	<version>1.13.2</version>
</dependency>

基于 Flink 服务提交任务并执行时需要的依赖包

基于 flink 服务器提交任务前，先上传依赖包到 flink 的 lib 目录下；然后重启 flink 服务，使 jar 进行加载；否则会出现 ClassNoFoundException 的异常。

• flink-connector-rabbitmq_2.12-1.13.2.jar
• amqp-client-5.9.0.jar

构建RabbitMQSink参数实例

public class RabbitSink implements Serializable {
	
	private static final long serialVersionUID = -6108637207745123361L;
	
	private String host; 
	
	private int port; 
	
	private String username; 
	
	private String password; 
	
	private String virtualHost;
	
	private String data;
	
	public String getHost() {
		return host;
	}
	
	public int getPort() {
		return port;
	}

	public String getUsername() {
		return username;
	}

	public String getPassword() {
		return password;
	}

	public String getVirtualHost() {
		return virtualHost;
	}

	public String getData() {
		return data;
	}
	
	public RabbitSink(Object obj) {
		final JSONObject json = JSONObject.parseObject(obj.toString());
		this.host = json.getString("host");
		this.port = json.getIntValue("port");
		this.username = json.getString("username");
		this.password = json.getString("password");
		this.virtualHost = json.getString("virtualHost");
		if(json.containsKey("data")) {
			this.data = json.getString("data");
	}
}

构建自定义RabbitMQSink

基于继承 RichSinkFunction< T > 抽象类实现自定义Sink，实现方法有三个：

• open()：构建sink节点时最先执行的方法，用于实现一些初始化动作。
• invoke()：执行节点时执行，用于实现具体业务逻辑。
• close()：关闭节点回收资源时执行，用于资源的回收。

import java.nio.charset.StandardCharsets;
import java.util.Map;

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;

import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;
import com.ygsoft.dataprocess.vo.sink.RabbitSink;

/**
 * rabbitmq sink 初始化
 * @author yinlilan
 *
 */
public class RabbitPropertySink extends RichSinkFunction<Map<String, String>> {

	private static final long serialVersionUID = -5295577568179093123L;

	final private RabbitSink rabbitSink;
	
	private Connection connection;
	
	private Channel channel;
	
	public RabbitPropertySink(final RabbitSink rabbitSink) {
		this.rabbitSink = rabbitSink;
	}
	
	@Override
	public void open(final Configuration parameters) throws Exception {
		super.open(parameters);
		// 创建连接工程
		ConnectionFactory factory = new ConnectionFactory();
		
		 // 设置RabbitMQ相关信息
        factory.setHost(rabbitSink.getHost());
        factory.setUsername(rabbitSink.getUsername());
        factory.setPassword(rabbitSink.getPassword());
        factory.setPort(rabbitSink.getPort());
		factory.setVirtualHost(rabbitSink.getVirtualHost());
		
		connection = factory.newConnection();
	}
	
	@Override
	public void invoke(Map<String, String> value, Context context) throws Exception {
		final Channel channel = connection.createChannel();
		channel.exchangeDeclare(value.get("topic"), "fanout");
		channel.basicPublish(value.get("topic"), "", null, value.get("value").getBytes(StandardCharsets.UTF_8));
	}
	
	@Override
    public void close() throws Exception {
        super.close();
        channel.close();
        connection.close();
    }
}

/**
 * rabbitmq sink 初始化
 * @author yinlilan
 *
 */
public class RabbitEventSink extends RichSinkFunction<Map<String, String>> {

	private static final long serialVersionUID = 5630254967223858908L;

	final private RabbitSink rabbitSink;
	
	private Connection connection;
	
	private Channel channel;
	
	public RabbitEventSink(final RabbitSink rabbitSink) {
		this.rabbitSink = rabbitSink;
	}
	
	@Override
	public void open(final Configuration parameters) throws Exception {
		super.open(parameters);
		// 创建连接工程
		ConnectionFactory factory = new ConnectionFactory();
		
		 // 设置RabbitMQ相关信息
        factory.setHost(rabbitSink.getHost());
        factory.setUsername(rabbitSink.getUsername());
        factory.setPassword(rabbitSink.getPassword());
        factory.setPort(rabbitSink.getPort());
		factory.setVirtualHost(rabbitSink.getVirtualHost());
		
		connection = factory.newConnection();
	}
	
	@Override
	public void invoke(Map<String, String> value, Context context) throws Exception {
		final Channel channel = connection.createChannel();
		final String topic = value.get("topic").replace("property", "event");
		channel.exchangeDeclare(topic, "fanout");
		
		final JSONArray events = new JSONArray();
		final JSONObject event = JSONObject.parseObject(rabbitSink.getData());
		final JSONArray propertys = JSONArray.parseArray(value.get("value").toString());
		for(int i=0; i<propertys.size(); i++) {
			final JSONObject property = propertys.getJSONObject(i);
			event.put("productId", property.get("productId"));
			event.put("deviceCode", property.get("deviceCode"));
			if (property.containsKey("subDeviceId")) {
				event.put("subDeviceId", property.get("subDeviceId"));
			}
			event.put("time", new Date().getTime());
			events.add(event);
		}
		
		channel.basicPublish(topic, "", null, events.toString().getBytes(StandardCharsets.UTF_8));
	}
	
	@Override
    public void close() throws Exception {
        super.close();
        channel.close();
        connection.close();
    }
}