flink:从pulsar topic 中读取类对象并写入到新的topic

本文链接：https://blog.csdn.net/amadeus_liu2/article/details/137085345

package cn.edu.tju.test2;

import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.pulsar.FlinkPulsarSink;
import org.apache.flink.streaming.connectors.pulsar.FlinkPulsarSource;
import org.apache.flink.streaming.connectors.pulsar.config.RecordSchemaType;
import org.apache.flink.streaming.connectors.pulsar.internal.AvroDeser;
import org.apache.flink.streaming.connectors.pulsar.internal.AvroSer;
import org.apache.flink.streaming.util.serialization.PulsarDeserializationSchema;
import org.apache.flink.streaming.util.serialization.PulsarPrimitiveSchema;
import org.apache.flink.streaming.util.serialization.PulsarSerializationSchemaWrapper;

import java.util.Optional;
import java.util.Properties;

public class FlinkPulsar03 {
    private static String SERVER_4 = "xx.xx.xx.xx";
    public static void main(String[] args) throws Exception {
        //创建flink的流式执行环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
        //配置pulsar
        String serviceUrl = "pulsar://" + SERVER_4 + ":6650";
        String adminUrl = "http://" + SERVER_4 + ":8080";
        Properties properties = new Properties();
        properties.put("topic", "persistent://public/default/my-topic3");

        //创建flink source
        FlinkPulsarSource<UserInfo> source = new FlinkPulsarSource<>(serviceUrl, adminUrl,
                AvroDeser.of(UserInfo.class), properties);

        //从topic 最早的数据开始消费
        source.setStartFromEarliest();
        DataStreamSource<UserInfo> dataStreamSource = env.addSource(source);

        PulsarSerializationSchemaWrapper<UserInfo> wrapper
                = new PulsarSerializationSchemaWrapper.Builder<>(AvroSer.of(UserInfo.class)
        ).usePojoMode(UserInfo.class, RecordSchemaType.AVRO).build();



        //将从my-topic3中读到的UserInfo类的对象写到my-topic5中
        FlinkPulsarSink<UserInfo> flinkPulsarSink = new FlinkPulsarSink<>(serviceUrl, adminUrl,
                Optional.of("my-topic5"), properties, wrapper);
        dataStreamSource.addSink(flinkPulsarSink);

        //执行job
        env.execute("my job");
    }


    private static class UserInfo{
        private String username;
        private String password;

        public String getUsername() {
            return username;
        }

        public void setUsername(String username) {
            this.username = username;
        }

        public String getPassword() {
            return password;
        }

        public void setPassword(String password) {
            this.password = password;
        }

        public UserInfo(String username, String password) {
            this.username = username;
            this.password = password;
        }

        public UserInfo() {
        }
        @Override
        public String toString() {
            return "UserInfo{" +
                    "username='" + username + '\'' +
                    ", password='" + password + '\'' +
                    '}';
        }

    }
}

完整依赖如下：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>pulsar-demo</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <pulsar.version>3.2.0</pulsar.version>
    </properties>

    <!-- in your <properties> block -->


    <!-- in your <dependencies> block -->

    <dependencies>
        <dependency>
            <groupId>org.apache.pulsar</groupId>
            <artifactId>pulsar-client-all</artifactId>
            <version>${pulsar.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-java</artifactId>
            <version>1.13.6</version>
        </dependency>



        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-java_2.11</artifactId>
            <version>1.13.6</version>
        </dependency>


        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_2.11</artifactId>
            <version>1.13.6</version>
        </dependency>

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_2.11</artifactId>
            <version>1.13.6</version>
        </dependency>




        <!-- https://mvnrepository.com/artifact/io.streamnative.connectors/pulsar-flink-connector -->
        <dependency>
            <groupId>io.streamnative.connectors</groupId>
            <artifactId>pulsar-flink-connector_2.11</artifactId>
            <version>1.13.6.1-rc13</version>
            <exclusions>
                <exclusion>
                    <groupId>org.apache.pulsar</groupId>
                    <artifactId>pulsar-client-all</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

    </dependencies>

</project>