Flink消费Rabbitmq匿名队列的数据（只需要绑定exchange和router-key）

Flink消费Rabbitmq匿名队列的数据（只需要绑定exchange和router-key）

1. 环境准备（使用的docker中的rabbitmq）

rabbitmq

# 查找rabbitmq应用
➜  ~ sudo docker search rabbitmq
# 下载镜像 (有时候网络问题超时，多尝试几次即可)
➜  ~ sudo docker pull rabbitmq：management
# 创建容器并运行(15672是管理界面的端口，5672是服务的端口。)
➜  ~ docker run -d -p 5672:5672 -p 15672:15672 --name myrabbitmq rabbitmq:management

2. JAVA代码

producer

import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.rabbitmq.client.ConnectionFactory;

import java.io.IOException;
import java.util.concurrent.TimeoutException;

public class MyProducer {
    public static void main(String[] args) throws IOException, TimeoutException {
        ConnectionFactory factory = new ConnectionFactory();
        factory.setHost("10.8.29.30");
        factory.setPort(5672);
        factory.setUsername("admin");
        factory.setPassword("admin");
        factory.setVirtualHost("/");
        Connection connection = factory.newConnection();
        Channel channel = connection.createChannel();
        String message = "hello message ";
        for (int i = 0; i < 100000; i++) {
            channel.basicPublish("amq.direct", "4xs-54mk5v2rVJ0KYZs_tA==", null, (message + i).getBytes());
        }
        channel.close();
        connection.close();
    }
}

main

import com.ruizhan.data.source.MySource;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.environment.LocalStreamEnvironment;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.rabbitmq.common.RMQConnectionConfig;


public class FlinkStreamingJob {
    public static void main(String[] args) throws Exception {
        LocalStreamEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
        env.enableCheckpointing(1000);
        // 线下
        RMQConnectionConfig config = new RMQConnectionConfig.Builder()
                .setHost("10.8.29.30")
                .setPort(5672)
                .setUserName("admin")
                .setPassword("admin")
                .setVirtualHost("/")
                .build();
        MySource<String> mySource = new MySource<String>(config, "", "amq.direct","4xs-54mk5v2rVJ0KYZs_tA==", false,
                                                         new SimpleStringSchema()) {
        };
        // RMQSource<String> rmqSource = new RMQSource<>(config, "queuename", new SimpleStringSchema());
        env.addSource(mySource).map((MapFunction<String, String>) value -> value).print();
        env.execute();
    }
}

MySource

flink消费rabbitmq有官方提供的RMQSource，但是使用这个的前提是明确队列名，但是业务上使用的队列是默认生成的，队列名类似“amq.gen-KiQAO2YsoywQX2kew2DDsg”，只能根据提供的direct和router-key来消费数据

import com.rabbitmq.client.Channel;
import com.rabbitmq.client.Connection;
import com.ruizhan.data.consumer.MyConsumer;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.serialization.DeserializationSchema;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.source.MultipleIdsMessageAcknowledgingSourceBase;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
import org.apache.flink.streaming.connectors.rabbitmq.common.RMQConnectionConfig;
import org.apache.flink.util.Collector;
import org.apache.flink.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.List;

public class MySource<OUT> extends MultipleIdsMessageAcknowledgingSourceBase<OUT, String, Long>
        implements ResultTypeQueryable<OUT> {

    private final RMQConnectionConfig rmqConnectionConfig;

    private static final long serialVersionUID = 1L;

    private static final Logger LOG = LoggerFactory.getLogger(MySource.class);

    protected String queueName;
    private final boolean usesCorrelationId;
    protected DeserializationSchema<OUT> schema;

    protected Connection connection;
    protected Channel channel;
    protected MyConsumer consumer;
    protected String exchange;
    protected String routerKey;

    protected transient boolean autoAck;

    private transient volatile boolean running;

    public MySource(RMQConnectionConfig rmqConnectionConfig, String queueName, String exchange, String routerKey,
                    DeserializationSchema<OUT> deserializationSchema) {
        this(rmqConnectionConfig, queueName, exchange, routerKey, false, deserializationSchema);
    }

    public MySource(RMQConnectionConfig rmqConnectionConfig,
                    String queueName, String exchange, String routerKey, boolean usesCorrelationId,
                    DeserializationSchema<OUT> deserializationSchema) {
        super(String.class);
        this.rmqConnectionConfig = rmqConnectionConfig;
        this.queueName = queueName;
        this.exchange = exchange;
        this.routerKey = routerKey;
        this.usesCorrelationId = usesCorrelationId;
        this.schema = deserializationSchema;
    }

    @Override
    public void open(Configuration config) throws Exception {
        super.open(config);
        try {
            connection = rmqConnectionConfig.getConnectionFactory().newConnection();
            channel = connection.createChannel();
            String queue = channel.queueDeclare().getQueue();
            System.out.println(queue);
            if (channel == null) {
                throw new RuntimeException("None of RabbitMQ channels are available");
            }
            //setupQueue();
            consumer = new MyConsumer(channel);

            RuntimeContext runtimeContext = getRuntimeContext();
            if (runtimeContext instanceof StreamingRuntimeContext
                    && ((StreamingRuntimeContext) runtimeContext).isCheckpointingEnabled()) {
                autoAck = false;
                // enables transaction mode
                channel.txSelect();
            } else {
                autoAck = true;
            }

            LOG.debug("Starting RabbitMQ source with autoAck status: " + autoAck);
            channel.queueBind(queue, this.exchange, this.routerKey);
//            channel.queueBind(queue, "mfp.exchange.distribute.FaceCapture", "4xs-54mk5v2rVJ0KYZs_tA==");
            channel.basicConsume(queue, false, consumer);

        } catch (IOException e) {
            throw new RuntimeException("Cannot create RMQ connection with " + queueName + " at "
                                               + rmqConnectionConfig.getHost(), e);
        }
        this.schema.open(() -> getRuntimeContext().getMetricGroup().addGroup("user"));
        running = true;
    }

    @Override
    public void close() throws Exception {
        super.close();

        try {
            if (consumer != null && channel != null) {
                channel.basicCancel(consumer.getConsumerTag());
            }
        } catch (IOException e) {
            throw new RuntimeException("Error while cancelling RMQ consumer on " + queueName
                                               + " at " + rmqConnectionConfig.getHost(), e);
        }

        try {
            if (channel != null) {
                channel.close();
            }
        } catch (IOException e) {
            throw new RuntimeException("Error while closing RMQ channel with " + queueName
                                               + " at " + rmqConnectionConfig.getHost(), e);
        }

        try {
            if (connection != null) {
                connection.close();
            }
        } catch (IOException e) {
            throw new RuntimeException("Error while closing RMQ connection with " + queueName
                                               + " at " + rmqConnectionConfig.getHost(), e);
        }
    }

    @Override
    public void run(SourceContext<OUT> ctx) throws Exception {
        final MySource.RMQCollector collector =
                new MySource.RMQCollector(ctx);
        while (running) {
            MyConsumer.Delivery delivery = consumer.nextDelivery();

            synchronized (ctx.getCheckpointLock()) {
                if (!autoAck) {
                    final long deliveryTag = delivery.getEnvelope().getDeliveryTag();
                    if (usesCorrelationId) {
                        final String correlationId = delivery.getProperties().getCorrelationId();
                        Preconditions.checkNotNull(correlationId, "RabbitMQ source was instantiated " +
                                "with usesCorrelationId set to true but a message was received with " +
                                "correlation id set to null!");
                        if (!addId(correlationId)) {
                            // we have already processed this message
                            continue;
                        }
                    }
                    sessionIds.add(deliveryTag);
                }
				// 数据下放的重点
                schema.deserialize(delivery.getBody(), collector);
                if (collector.isEndOfStreamSignalled()) {
                    this.running = false;
                    return;
                }
            }
        }
    }

    private class RMQCollector implements Collector<OUT> {

        private final SourceContext<OUT> ctx;
        private boolean endOfStreamSignalled = false;

        private RMQCollector(SourceContext<OUT> ctx) {
            this.ctx = ctx;
        }

        @Override
        public void collect(OUT record) {
            if (endOfStreamSignalled || schema.isEndOfStream(record)) {
                this.endOfStreamSignalled = true;
                return;
            }

            ctx.collect(record);
        }

        public boolean isEndOfStreamSignalled() {
            return endOfStreamSignalled;
        }

        @Override
        public void close() {

        }
    }

    @Override
    public void cancel() {
        running = false;
    }

    @Override
    protected void acknowledgeSessionIDs(List<Long> sessionIds) {
        try {
            for (long id : sessionIds) {
                channel.basicAck(id, false);
            }
            channel.txCommit();
        } catch (IOException e) {
            throw new RuntimeException("Messages could not be acknowledged during checkpoint creation.", e);
        }
    }

    @Override
    public TypeInformation<OUT> getProducedType() {
        return schema.getProducedType();
    }

}

MyConsumer

这里使用自己的消费者而没有使用原本的消费者只是为了方便自己debugger时，确认数据是否正确到达消费端

import com.rabbitmq.client.*;
import com.rabbitmq.utility.Utility;

import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;

public class MyConsumer extends DefaultConsumer {
    private final BlockingQueue<MyConsumer.Delivery> _queue;

    private volatile ShutdownSignalException _shutdown;
    private volatile ConsumerCancelledException _cancelled;

    private static final MyConsumer.Delivery POISON =
            new MyConsumer.Delivery(null, null, null);

    public MyConsumer(Channel ch) {
        this(ch, new LinkedBlockingQueue<>());
    }

    public MyConsumer(Channel ch, BlockingQueue<MyConsumer.Delivery> q) {
        super(ch);
        this._queue = q;
    }

    @Override public void handleShutdownSignal(String consumerTag,
                                               ShutdownSignalException sig) {
        _shutdown = sig;
        _queue.add(POISON);
    }

    @Override public void handleCancel(String consumerTag) {
        _cancelled = new ConsumerCancelledException();
        _queue.add(POISON);
    }

    @Override
    public void handleDelivery(String consumerTag,
                                         Envelope envelope,
                                         AMQP.BasicProperties properties,
                                         byte[] body) {
        checkShutdown();
        this._queue.add(new MyConsumer.Delivery(envelope, properties, body));
    }

    /**
     * Encapsulates an arbitrary message - simple "bean" holder structure.
     */
    public static class Delivery {
        private final Envelope _envelope;
        private final AMQP.BasicProperties _properties;
        private final byte[] _body;

        public Delivery(Envelope envelope, AMQP.BasicProperties properties, byte[] body) {
            _envelope = envelope;
            _properties = properties;
            _body = body;
        }

        /**
         * Retrieve the message envelope.
         * @return the message envelope
         */
        public Envelope getEnvelope() {
            return _envelope;
        }

        /**
         * Retrieve the message properties.
         * @return the message properties
         */
        public AMQP.BasicProperties getProperties() {
            return _properties;
        }

        /**
         * Retrieve the message body.
         * @return the message body
         */
        public byte[] getBody() {
            return _body;
        }
    }

    /**
     * Check if we are in shutdown mode and if so throw an exception.
     */
    private void checkShutdown() {
        if (_shutdown != null)
            throw Utility.fixStackTrace(_shutdown);
    }

    /**
     * If delivery is not POISON nor null, return it.
     * <p/>
     * If delivery, _shutdown and _cancelled are all null, return null.
     * <p/>
     * If delivery is POISON re-insert POISON into the queue and
     * throw an exception if POISONed for no reason.
     * <p/>
     * Otherwise, if we are in shutdown mode or cancelled,
     * throw a corresponding exception.
     */
    private MyConsumer.Delivery handle(MyConsumer.Delivery delivery) {
        if (delivery == POISON ||
                delivery == null && (_shutdown != null || _cancelled != null)) {
            if (delivery == POISON) {
                _queue.add(POISON);
                if (_shutdown == null && _cancelled == null) {
                    throw new IllegalStateException(
                            "POISON in queue, but null _shutdown and null _cancelled. " +
                                    "This should never happen, please report as a BUG");
                }
            }
        }
        return delivery;
    }

    /**
     * Main application-side API: wait for the next message delivery and return it.
     * @return the next message
     * @throws InterruptedException if an interrupt is received while waiting
     * @throws ShutdownSignalException if the connection is shut down while waiting
     * @throws ConsumerCancelledException if this consumer is cancelled while waiting
     */
    public MyConsumer.Delivery nextDelivery()
            throws InterruptedException, ShutdownSignalException, ConsumerCancelledException
    {
        return handle(_queue.take());
    }

}