项目场景:
`flink 接入rabbitmq 数据源,用的是topic 方式进行数据接受,但是发现flink提供的RMQSource无法进行交换器和路由的绑定,这就有点限制操作了
RMQSource
public void open(Configuration config) throws Exception {
super.open(config);
try {
this.connection = this.setupConnection();
this.channel = this.setupChannel(this.connection);
if (this.channel == null) {
throw new RuntimeException("None of RabbitMQ channels are available");
}
this.setupQueue();
this.consumer = new QueueingConsumer(this.channel);
RuntimeContext runtimeContext = this.getRuntimeContext();
if (runtimeContext instanceof StreamingRuntimeContext && ((StreamingRuntimeContext)runtimeContext).isCheckpointingEnabled()) {
this.autoAck = false;
this.channel.txSelect();
} else {
this.autoAck = true;
}
LOG.debug("Starting RabbitMQ source with autoAck status: " + this.autoAck);
this.channel.basicConsume(this.queueName, this.autoAck, this.consumer);
} catch (IOException var3) {
throw new RuntimeException("Cannot create RMQ connection with " + this.queueName + " at " + this.rmqConnectionConfig.getHost(), var3);
}
this.deliveryDeserializer.open(RuntimeContextInitializationContextAdapters.deserializationAdapter(this.getRuntimeContext(), (metricGroup) -> {
return metricGroup.addGroup("user");
}));
this.running = true;
}
解决方案:
模仿RMQSource,重新写自己的source
import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.MultipleIdsMessageAcknowledgingSourceBase;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
import org.apache.flink.streaming.connectors.rabbitmq.common.RMQConnectionConfig;
import org.apache.flink.util.Collector;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
public class RabbitMqSource<OUT> extends MultipleIdsMessageAcknowledgingSourceBase<OUT, String, Long>
implements ResultTypeQueryable<OUT> {
private final RMQConnectionConfig rmqConnectionConfig;
private static final long serialVersionUID = 1L;
private static final Logger LOG = LoggerFactory.getLogger(RabbitMqSource.class);
protected String queueName;
private final boolean usesCorrelationId;
protected DeserializationSchema<OUT> schema;
protected Connection connection;
protected Channel channel;
protected RabbitMqConsumer consumer;
protected String exchange;
protected String routerKey;
protected transient boolean autoAck;
private transient volatile boolean running;
public RabbitMqSource(RMQConnectionConfig rmqConnectionConfig, String queueName, String exchange, String routerKey,
DeserializationSchema<OUT> deserializationSchema) {
this(rmqConnectionConfig, queueName, exchange, routerKey, false, deserializationSchema);
}
public RabbitMqSource(RMQConnectionConfig rmqConnectionConfig,
String queueName, String exchange, String routerKey, boolean usesCorrelationId,
DeserializationSchema<OUT> deserializationSchema) {
super(String.class);
this.rmqConnectionConfig = rmqConnectionConfig;
this.queueName = queueName;
this.exchange = exchange;
this.routerKey = routerKey;
this.usesCorrelationId = usesCorrelationId;
this.schema = deserializationSchema;
}
@Override
public void open(Configuration config) throws Exception {
super.open(config);
try {
connection = rmqConnectionConfig.getConnectionFactory().newConnection();
channel = connection.createChannel();
String queue = channel.queueDeclare(queueName, false, false, true, null).getQueue();
if (channel == null) {
throw new RuntimeException("None of RabbitMQ channels are available");
}
consumer = new RabbitMqConsumer(channel);
RuntimeContext runtimeContext = getRuntimeContext();
if (runtimeContext instanceof StreamingRuntimeContext
&& ((StreamingRuntimeContext) runtimeContext).isCheckpointingEnabled()) {
autoAck = false;
// enables transaction mode
channel.txSelect();
} else {
autoAck = true;
}
LOG.debug("Starting RabbitMQ source with autoAck status: " + autoAck);
channel.queueBind(queue, this.exchange, this.routerKey);
channel.basicConsume(queue, this.autoAck, consumer);
} catch (IOException e) {
throw new RuntimeException("Cannot create RMQ connection with " + queueName + " at "
+ rmqConnectionConfig.getHost(), e);
}
running = true;
}
@Override
public void close() throws Exception {
super.close();
try {
if (consumer != null && channel != null) {
channel.basicCancel(consumer.getConsumerTag());
}
} catch (IOException e) {
throw new RuntimeException("Error while cancelling RMQ consumer on " + queueName
+ " at " + rmqConnectionConfig.getHost(), e);
}
try {
if (channel != null) {
channel.close();
}
} catch (IOException e) {
throw new RuntimeException("Error while closing RMQ channel with " + queueName
+ " at " + rmqConnectionConfig.getHost(), e);
}
try {
if (connection != null) {
connection.close();
}
} catch (IOException e) {
throw new RuntimeException("Error while closing RMQ connection with " + queueName
+ " at " + rmqConnectionConfig.getHost(), e);
}
}
@Override
public void run(SourceContext<OUT> ctx) throws Exception {
final RabbitMqSource.RMQCollector collector =
new RabbitMqSource.RMQCollector(ctx);
while (running) {
RabbitMqConsumer.Delivery delivery = consumer.nextDelivery();
if(null == delivery){
continue;
}
synchronized (ctx.getCheckpointLock()) {
if (!autoAck) {
final long deliveryTag = delivery.getEnvelope().getDeliveryTag();
if (usesCorrelationId) {
final String correlationId = delivery.getProperties().getCorrelationId();
Preconditions.checkNotNull(correlationId, "RabbitMQ source was instantiated " +
"with usesCorrelationId set to true but a message was received with " +
"correlation id set to null!");
if (!addId(correlationId)) {
// we have already processed this message
continue;
}
}
sessionIds.add(deliveryTag);
}
try {
schema.deserialize(delivery.getBody(), collector);
}catch (Exception e){
System.out.println("RabbitMqSource.run"+e);
}
if (collector.isEndOfStreamSignalled()) {
this.running = false;
return;
}
}
}
}
private class RMQCollector implements Collector<OUT> {
private final SourceContext<OUT> ctx;
private boolean endOfStreamSignalled = false;
private RMQCollector(SourceContext<OUT> ctx) {
this.ctx = ctx;
}
@Override
public void collect(OUT record) {
if (endOfStreamSignalled || schema.isEndOfStream(record)) {
this.endOfStreamSignalled = true;
return;
}
ctx.collect(record);
}
public boolean isEndOfStreamSignalled() {
return endOfStreamSignalled;
}
@Override
public void close() {
}
}
@Override
public void cancel() {
running = false;
}
@Override
protected void acknowledgeSessionIDs(List<Long> sessionIds) {
try {
for (long id : sessionIds) {
channel.basicAck(id, false);
}
channel.txCommit();
} catch (IOException e) {
throw new RuntimeException("Messages could not be acknowledged during checkpoint creation.", e);
}
}
@Override
public TypeInformation<OUT> getProducedType() {
return schema.getProducedType();
}
}