Flink 链接rabbitmq 绑定exchange和routerKey

项目场景:

`flink 接入rabbitmq 数据源,用的是topic 方式进行数据接受,但是发现flink提供的RMQSource无法进行交换器和路由的绑定,这就有点限制操作了

RMQSource

  public void open(Configuration config) throws Exception {
        super.open(config);

        try {
            this.connection = this.setupConnection();
            this.channel = this.setupChannel(this.connection);
            if (this.channel == null) {
                throw new RuntimeException("None of RabbitMQ channels are available");
            }

            this.setupQueue();
            this.consumer = new QueueingConsumer(this.channel);
            RuntimeContext runtimeContext = this.getRuntimeContext();
            if (runtimeContext instanceof StreamingRuntimeContext && ((StreamingRuntimeContext)runtimeContext).isCheckpointingEnabled()) {
                this.autoAck = false;
                this.channel.txSelect();
            } else {
                this.autoAck = true;
            }

            LOG.debug("Starting RabbitMQ source with autoAck status: " + this.autoAck);
            this.channel.basicConsume(this.queueName, this.autoAck, this.consumer);
        } catch (IOException var3) {
            throw new RuntimeException("Cannot create RMQ connection with " + this.queueName + " at " + this.rmqConnectionConfig.getHost(), var3);
        }

        this.deliveryDeserializer.open(RuntimeContextInitializationContextAdapters.deserializationAdapter(this.getRuntimeContext(), (metricGroup) -> {
            return metricGroup.addGroup("user");
        }));
        this.running = true;
    }


解决方案:

模仿RMQSource,重新写自己的source

import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.MultipleIdsMessageAcknowledgingSourceBase;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
import org.apache.flink.streaming.connectors.rabbitmq.common.RMQConnectionConfig;
import org.apache.flink.util.Collector;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.List;

public class RabbitMqSource<OUT> extends MultipleIdsMessageAcknowledgingSourceBase<OUT, String, Long>
        implements ResultTypeQueryable<OUT> {

    private final RMQConnectionConfig rmqConnectionConfig;

    private static final long serialVersionUID = 1L;

    private static final Logger LOG = LoggerFactory.getLogger(RabbitMqSource.class);

    protected String queueName;
    private final boolean usesCorrelationId;
    protected DeserializationSchema<OUT> schema;

    protected Connection connection;
    protected Channel channel;
    protected RabbitMqConsumer consumer;
    protected String exchange;
    protected String routerKey;

    protected transient boolean autoAck;

    private transient volatile boolean running;

    public RabbitMqSource(RMQConnectionConfig rmqConnectionConfig, String queueName, String exchange, String routerKey,
                          DeserializationSchema<OUT> deserializationSchema) {
        this(rmqConnectionConfig, queueName, exchange, routerKey, false, deserializationSchema);
    }

    public RabbitMqSource(RMQConnectionConfig rmqConnectionConfig,
                          String queueName, String exchange, String routerKey, boolean usesCorrelationId,
                          DeserializationSchema<OUT> deserializationSchema) {
        super(String.class);
        this.rmqConnectionConfig = rmqConnectionConfig;
        this.queueName = queueName;
        this.exchange = exchange;
        this.routerKey = routerKey;
        this.usesCorrelationId = usesCorrelationId;
        this.schema = deserializationSchema;
    }

    @Override
    public void open(Configuration config) throws Exception {
        super.open(config);
        try {
            connection = rmqConnectionConfig.getConnectionFactory().newConnection();
            channel = connection.createChannel();
            String queue = channel.queueDeclare(queueName, false, false, true, null).getQueue();
            if (channel == null) {
                throw new RuntimeException("None of RabbitMQ channels are available");
            }

            consumer = new RabbitMqConsumer(channel);

            RuntimeContext runtimeContext = getRuntimeContext();
            if (runtimeContext instanceof StreamingRuntimeContext
                    && ((StreamingRuntimeContext) runtimeContext).isCheckpointingEnabled()) {
                autoAck = false;
                // enables transaction mode
                channel.txSelect();
            } else {
                autoAck = true;
            }

            LOG.debug("Starting RabbitMQ source with autoAck status: " + autoAck);
            channel.queueBind(queue, this.exchange, this.routerKey);

            channel.basicConsume(queue, this.autoAck, consumer);

        } catch (IOException e) {
            throw new RuntimeException("Cannot create RMQ connection with " + queueName + " at "
                                               + rmqConnectionConfig.getHost(), e);
        }

        running = true;
    }

    @Override
    public void close() throws Exception {
        super.close();

        try {
            if (consumer != null && channel != null) {
                channel.basicCancel(consumer.getConsumerTag());
            }
        } catch (IOException e) {
            throw new RuntimeException("Error while cancelling RMQ consumer on " + queueName
                                               + " at " + rmqConnectionConfig.getHost(), e);
        }

        try {
            if (channel != null) {
                channel.close();
            }
        } catch (IOException e) {
            throw new RuntimeException("Error while closing RMQ channel with " + queueName
                                               + " at " + rmqConnectionConfig.getHost(), e);
        }

        try {
            if (connection != null) {
                connection.close();
            }
        } catch (IOException e) {
            throw new RuntimeException("Error while closing RMQ connection with " + queueName
                                               + " at " + rmqConnectionConfig.getHost(), e);
        }
    }

    @Override
    public void run(SourceContext<OUT> ctx) throws Exception {
        final RabbitMqSource.RMQCollector collector =
                new RabbitMqSource.RMQCollector(ctx);
        while (running) {
            RabbitMqConsumer.Delivery delivery = consumer.nextDelivery();
            if(null == delivery){
                continue;
            }

            synchronized (ctx.getCheckpointLock()) {
                if (!autoAck) {
                    final long deliveryTag = delivery.getEnvelope().getDeliveryTag();
                    if (usesCorrelationId) {
                        final String correlationId = delivery.getProperties().getCorrelationId();
                        Preconditions.checkNotNull(correlationId, "RabbitMQ source was instantiated " +
                                "with usesCorrelationId set to true but a message was received with " +
                                "correlation id set to null!");
                        if (!addId(correlationId)) {
                            // we have already processed this message
                            continue;
                        }
                    }
                    sessionIds.add(deliveryTag);
                }
                try {

                    schema.deserialize(delivery.getBody(), collector);
                }catch (Exception e){
                    System.out.println("RabbitMqSource.run"+e);
                }
                if (collector.isEndOfStreamSignalled()) {
                    this.running = false;
                    return;
                }
            }
        }
    }

    private class RMQCollector implements Collector<OUT> {

        private final SourceContext<OUT> ctx;
        private boolean endOfStreamSignalled = false;

        private RMQCollector(SourceContext<OUT> ctx) {
            this.ctx = ctx;
        }

        @Override
        public void collect(OUT record) {
            if (endOfStreamSignalled || schema.isEndOfStream(record)) {
                this.endOfStreamSignalled = true;
                return;
            }

            ctx.collect(record);
        }

        public boolean isEndOfStreamSignalled() {
            return endOfStreamSignalled;
        }

        @Override
        public void close() {

        }
    }

    @Override
    public void cancel() {
        running = false;
    }

    @Override
    protected void acknowledgeSessionIDs(List<Long> sessionIds) {
        try {
            for (long id : sessionIds) {
                channel.basicAck(id, false);
            }
            channel.txCommit();
        } catch (IOException e) {
            throw new RuntimeException("Messages could not be acknowledged during checkpoint creation.", e);
        }
    }

    @Override
    public TypeInformation<OUT> getProducedType() {
        return schema.getProducedType();
    }

}

  • 1
    点赞
  • 3
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值