Flink1.12-RocketMQ4.5.1-Source

Flink1.12-RocketMQ4.5.1-Source

KeyValueDeserializationSchema

package com.flink_rmq_source;

import org.apache.flink.api.java.typeutils.ResultTypeQueryable;

import java.io.Serializable;

public interface KeyValueDeserializationSchema<T> extends ResultTypeQueryable<T>, Serializable {
   
    T deserializeKeyAndValue(byte[] key, byte[] value);
}

MqStreamConsumer

package com.flink_rmq_source;

import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

import java.util.Properties;

public class MqStreamConsumer {
   
    //nameserver.address
    static String  address="hadoop104:9876";
    static String topic="jack-topic";
    static String group="jack-producer-group";
    static String tag="jack-tag";
    static String offset="";

    public static void main(String[] args) throws Exception {
   


        final StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
        DataStream<String> text = env.addSource(builderTurboMQSource()).name("turboMQSource");
        text.print();
        env.execute("MqStreamConsumer");
    }


    private static RocketMQSource<String> builderTurboMQSource() {
   
        Properties props = new Properties();
        props.setProperty(RocketMQConfig.NAME_SERVER_ADDR, address);
        props.setProperty(RocketMQConfig.CONSUMER_TOPIC, topic);
        props.setProperty(RocketMQConfig.CONSUMER_GROUP, group);
        props.setProperty(RocketMQConfig.CONSUMER_TAG, tag);
        props.setProperty(RocketMQConfig.CONSUMER_OFFSET_RESET_TO, offset);

        KeyValueDeserializationSchema<String> schema = new SimpleStringDeserializationSchema();

        RocketMQSource<String> turboMQConsumer = new RocketMQSource<String>(schema, props);
        return turboMQConsumer;
    }
}

Producer

package com.flink_rmq_source;

import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.client.producer.SendResult;
import org.apache.rocketmq.common.message.Message;

public class Producer {
   
    public static void main(String[] args) throws Exception {
   
        // TODO 1 创建消息生产者，指定生成组名
        DefaultMQProducer defaultMQProducer = new DefaultMQProducer("jack-producer-group");
        // TODO 2 指定NameServer的地址
        defaultMQProducer.setNamesrvAddr("hadoop104:9876");
        // TODO 3 启动生产者
        defaultMQProducer.start();
        // TODO 4 构建消息对象，主要是设置消息的主题、标签、内容
        Message message = new Message("jack-topic", "jack-tag", ("测试消息发送 --Jack").getBytes());
        // TODO 5 发送消息
        SendResult result = defaultMQProducer.send(message);
        System.out.println("SendResult-->" + result);
        // TODO 6 关闭生产者
        defaultMQProducer.shutdown();
    }


}

RocketMQConfig

package com.flink_rmq_source;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.Validate;
import org.apache.rocketmq.client.ClientConfig;
import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.common.protocol.heartbeat.MessageModel;

import java.util.Properties;
import java.util.UUID;

import static com.flink_rmq_source.RocketMQUtils.getInteger;

/**
 * RocketMQConfig for Consumer/Producer.
 */
public class RocketMQConfig {
   
    // Server Config
    public static final String NAME_SERVER_ADDR = "nameserver.address"; // Required

    public static final String NAME_SERVER_POLL_INTERVAL = "nameserver.poll.interval";
    public static final int DEFAULT_NAME_SERVER_POLL_INTERVAL = 30000; // 30 seconds

    public static final String BROKER_HEART_BEAT_INTERVAL = "brokerserver.heartbeat.interval";
    public static final int DEFAULT_BROKER_HEART_BEAT_INTERVAL = 30000; // 30 seconds


    // Producer related config
    public static final String PRODUCER_GROUP = "producer.group";

    public static final String PRODUCER_RETRY_TIMES = "producer.retry.times";
    public static final int DEFAULT_PRODUCER_RETRY_TIMES = 3;

    public static final String PRODUCER_TIMEOUT = "producer.timeout";
    public static final int DEFAULT_PRODUCER_TIMEOUT = 3000; // 3 seconds


    // Consumer related config
    public static final String CONSUMER_GROUP = "consumer.group"; // Required

    public static final String CONSUMER_TOPIC = "consumer.topic"; // Required

    public static final String CONSUMER_TAG = "consumer.tag";
    public static final String DEFAULT_CONSUMER_TAG = "*";

    public static final String CONSUMER_OFFSET_RESET_TO = "consumer.offset.reset.to";
    public static final String CONSUMER_OFFSET_LATEST = "latest";
    public static final String CONSUMER_OFFSET_EARLIEST = "earliest";
    public static final String CONSUMER_OFFSET_TIMESTAMP = "timestamp";
    public static final String CONSUMER_OFFSET_FROM_TIMESTAMP = "consumer.offset.from.timestamp";

    public static final String CONSUMER_OFFSET_PERSIST_INTERVAL = "consumer.offset.persist.interval";
    public static final int DEFAULT_CONSUMER_OFFSET_PERSIST_INTERVAL = 5000; // 5 seconds

    public static final String CONSUMER_PULL_POOL_SIZE = "consumer.pull.thread.pool.size";
    public static final int DEFAULT_CONSUMER_PULL_POOL_SIZE = 20;

    public static final String CONSUMER_BATCH_SIZE = "consumer.batch.size";
    public static final int DEFAULT_CONSUMER_BATCH_SIZE = 32;

    public static final String CONSUMER_DELAY_WHEN_MESSAGE_NOT_FOUND = "consumer.delay.when.message.not.found";
    public static final int DEFAULT_CONSUMER_DELAY_WHEN_MESSAGE_NOT_FOUND = 10;

    public static final String MSG_DELAY_LEVEL = "msg.delay.level";
    public static final int MSG_DELAY_LEVEL00 = 0; // no delay
    public static final int MSG_DELAY_LEVEL01 = 1; // 1s
    public static final int MSG_DELAY_LEVEL02 = 2; // 5s
    public static final int MSG_DELAY_LEVEL03 = 3; // 10s
    public static final int MSG_DELAY_LEVEL04 = 4; // 30s
    public static final int MSG_DELAY_LEVEL05 = 5; // 1min
    public static final int MSG_DELAY_LEVEL06 = 6; // 2min
    public static final int MSG_DELAY_LEVEL07 = 7; // 3min
    public static final int MSG_DELAY_LEVEL08 = 8; // 4min
    public static final int MSG_DELAY_LEVEL09 = 9; // 5min
    public static final int MSG_DELAY_LEVEL10 = 10;