Flink学习:Flink Connecter连接各数据源

最新推荐文章于 2023-09-06 16:54:19 发布
最新推荐文章于 2023-09-06 16:54:19 发布
阅读量572 收藏 2
点赞数 1
分类专栏： Flink 文章标签： flink 学习 大数据
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/nzbing/article/details/127714182
版权

Flink

一、代码实现

<一>、读取Json数据,写入HDFS

package cn.flink.opt1;

import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableEnvironment;

public class FlinkJson2HDFSCsv {

    public static void main(String[] args) {
        //获取tableEnvironment

        EnvironmentSettings environmentSettings = EnvironmentSettings.newInstance()
                .build();
        TableEnvironment tEnv = TableEnvironment.create(environmentSettings);
	
		//读取本地input文件下json数据,文件自行准备
        String source_sql = "CREATE TABLE json_table (\n" +
                "  id Integer,\n" +
                "  name STRING,\n" +
                "  email STRING,\n" +
                "  date_time STRING" +
                ") WITH (\n" +
                "  'connector'='filesystem',\n" +
                "  'path'='input/userbase.json',\n" +
                "  'format'='json'\n" +
                ")";

        String sink_sql = "CREATE TABLE sink_hdfs (\n" +
                "  id Integer,\n" +
                "  name STRING,\n" +
                "  email STRING,\n" +
                "  date_time STRING" +
                ") WITH ( \n " +
                " 'connector' = 'filesystem',\n" +
                " 'path' = 'hdfs://localhost:9000/output_csv/' , \n" +
                " 'format' = 'csv'\n" +
                ")";
        //hdfs://localhost:9000,配置在core-site.xml下

        String insert_sql = "insert into sink_hdfs select id,name,date_time,email from json_table ";

        tEnv.executeSql(source_sql);
        tEnv.executeSql(sink_sql);
        tEnv.executeSql(insert_sql);

    }
}

在这里插入图片描述

<二>、读取HDFS数据,写入HBase

package cn.flink.opt1;

import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableEnvironment;

public class FlinkWithHDFSCSV2HBase {
    public static void main(String[] args) {
        //获取tableEnvironment

        EnvironmentSettings environmentSettings = EnvironmentSettings.newInstance()
                .build();
        TableEnvironment tEnv = TableEnvironment.create(environmentSettings);

        String source_sql = "CREATE TABLE source_hdfs (\n" +
                "  id Integer,\n" +
                "  name STRING,\n" +
                "  email STRING,\n" +
                "  date_time STRING" +
                ") WITH ( \n " +
                " 'connector' = 'filesystem',\n" +
                " 'path' = 'hdfs://localhost:9000/output_csv/' , \n" +
                " 'format' = 'csv'\n" +
                ")";

        String sink_sql = "CREATE TABLE sink_table (\n" +
                " rowkey Integer,\n" +
                " f1 ROW<name STRING,email STRING,date_time STRING > ,\n" +
                " PRIMARY KEY (rowkey) NOT ENFORCED \n" +
                ") WITH (\n" +
                " 'connector' = 'hbase-2.2',\n" +
                " 'table-name' = 'hTable',\n" +
                " 'zookeeper.quorum' = 'bigdata01:2181,bigdata02:2181,bigdata03:2181'\n" +
                ") ";


        String execute_sql = "insert into sink_table select id as rowkey ,ROW(name,email,date_time) from source_hdfs";

        tEnv.executeSql(source_sql);
        tEnv.executeSql(sink_sql);
        tEnv.executeSql(execute_sql);
    }
}

<三>、读取HBase数据写入Kafka

package cn.flink.opt1;

import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableEnvironment;

public class FlinkTableWithHBase2Kafka {
    public static void main(String[] args) {


        EnvironmentSettings environmentSettings = EnvironmentSettings.newInstance()
                .build();
        TableEnvironment tEnv = TableEnvironment.create(environmentSettings);



        String source_table = "CREATE TABLE opt_log (\n" +
                " rowkey Integer,\n" +
                " f1 ROW<username STRING,email STRING,date_time STRING > ,\n" +
                " PRIMARY KEY (rowkey) NOT ENFORCED \n" +
                ") WITH (\n" +
                " 'connector' = 'hbase-2.2',\n" +
                " 'table-name' = 'opt_log',\n" +
                " 'zookeeper.quorum' = 'bigdata01:2181,bigdata02:2181,bigdata03:2181'\n" +
                ") ";



        String sink_table = "CREATE TABLE KafkaTable (\n" +
                "  `username` STRING,\n" +
                "  `email` STRING,\n" +
                "  `date_time` STRING \n" +
                ") WITH (\n" +
                "  'connector' = 'kafka',\n" +
                "  'topic' = 'user_output',\n" +   //指定topic
                "  'properties.bootstrap.servers' = 'bigdata01:9092,bigdata02:9092,bigdata03:9092',\n" +
                "  'format' = 'json'\n" +
                ")";

        String execute_sql = "insert into KafkaTable select username,email,date_time from opt_log";

        tEnv.executeSql(source_table);
        tEnv.executeSql(sink_table);
        tEnv.executeSql(execute_sql);
    }
}

<四>、读取Kafka数据写入Mysql

package cn.flink.opt1;

import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.TableEnvironment;

public class FlinkTableWithKafka2MySQL {
    public static void main(String[] args) {

        EnvironmentSettings environmentSettings = EnvironmentSettings.newInstance()
                // .inStreamingMode()//默认就是这个
                //      .useBlinkPlanner()//flink1.14版本之后，默认就是使用了Blink
                .build();
        TableEnvironment tEnv = TableEnvironment.create(environmentSettings);


        String source_table = "CREATE TABLE KafkaTable (\n" +
                "  `id` Integer,\n" +
                "  `name` STRING,\n" +
                "  `email` STRING,\n" +
                "  `date_time` STRING \n" +
                ") WITH (\n" +
                "  'connector' = 'kafka',\n" +
                "  'topic' = 'usr_opt',\n" +
                "  'properties.bootstrap.servers' = 'bigdata01:9092,bigdata02:9092,bigdata03:9092',\n" +
                "  'properties.group.id' = 'user_opt_group',\n" +
                "  'scan.startup.mode' = 'earliest-offset',\n" +
                "  'format' = 'json',\n" +
                "   'json.fail-on-missing-field' = 'false',\n" +
                " 'json.ignore-parse-errors' = 'true'\n" +
                ")";


        String sink_sql = "CREATE TABLE mysql_sink (\n" +
                "  id Integer,\n" +
                "  username STRING,\n" +
                "  email STRING,\n" +
                "  date_time STRING" +
                ") WITH (\n" +
                "  'connector' = 'jdbc',\n" +
                "  'url' = 'jdbc:mysql://bigdata03:3306/user_log?characterEncoding=utf-8&serverTimezone=GMT%2B8',\n" +
                "  'driver' = 'com.mysql.jdbc.Driver',\n" +
                "  'table-name' = 'clicklog',\n" +
                "  'username' = 'root',\n" +
                "  'password' = '123456'\n" +
                ")";

        String execute_sql ="insert into mysql_sink select id , name as username,email,date_time from KafkaTable";

        tEnv.executeSql(source_table);
        tEnv.executeSql(sink_sql);
        tEnv.executeSql(execute_sql);
    }

}

二、pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>cn.flinksql</groupId>
    <artifactId>flink_sql_study</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <scala.binary.version>2.12</scala.binary.version>
        <flink.version>1.14.3</flink.version>
        <hadoop.version>3.1.4</hadoop.version>
        <hbase.version>2.2.7</hbase.version>
        <hive.version>3.1.2</hive.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!-- <scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!-- <scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-streaming-scala_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
            <!-- <scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-table-common</artifactId>
            <version>${flink.version}</version>
            <!-- <scope>provided</scope>-->
        </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-clients_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.22</version>
            <scope>provided</scope>
        </dependency>
        <!-- https://mvnrepository.com/artifact/com.alibaba/fastjson -->
        <dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.73</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>1.7.15</version>
        </dependency>

        <!-- flink 读写CSV文件  -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-csv</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!-- flink 读写json文件  -->

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-json</artifactId>
            <version>${flink.version}</version>
        </dependency>


     <!--  flink  读写HDFS需要用到的jar包  -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>

            <exclusions>
                <exclusion>
                    <groupId>org.apache.commons</groupId>
                    <artifactId>commons-math3</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.apache.commons</groupId>
                    <artifactId>commons-compress</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-auth</artifactId>
            <version>${hadoop.version}</version>
        </dependency>



        <dependency>
            <groupId>org.apache.commons</groupId>
            <artifactId>commons-math3</artifactId>
            <version>3.5</version>
        </dependency>


        <!--  flink 整合HBase需要用到的jar包-->


        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-hbase-2.2_2.12</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-server</artifactId>
            <version>${hbase.version}</version>
            <exclusions>
                <exclusion>
                    <artifactId>commons-math3</artifactId>
                    <groupId>org.apache.commons</groupId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-client</artifactId>
            <version>${hbase.version}</version>
        </dependency>

        <!--   flink整合Kafka需要用到的jar包 -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
            <version>${flink.version}</version>
        </dependency>

        <!-- flink读写jdbc需要用到的jar包 -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-jdbc_2.11</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.38</version>
        </dependency>

        <!-- flink整合hive需要用到的jar包  -->

        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-hive_2.12</artifactId>
            <version>${flink.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-exec</artifactId>
            <version>${hive.version}</version>
            <exclusions>
                <exclusion>
                    <artifactId>hadoop-hdfs</artifactId>
                    <groupId>org.apache.hadoop</groupId>
                </exclusion>
                <exclusion>
                    <artifactId>commons-compress</artifactId>
                    <groupId>org.apache.commons</groupId>
                </exclusion>
            </exclusions>
        </dependency>

        <!--  <dependency>
              <groupId>org.apache.commons</groupId>
              <artifactId>commons-io</artifactId>
              <version>1.3.2</version>
          </dependency>
  -->

        <!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>2.11.0</version>
        </dependency>

        <dependency>
            <groupId>org.antlr</groupId>
            <artifactId>antlr-runtime</artifactId>
            <version>3.5.2</version>
        </dependency>


        <dependency>
            <groupId>org.apache.thrift</groupId>
            <artifactId>libfb303</artifactId>
            <version>0.9.3</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.5.1</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.1.1</version>
                <configuration>
                    <!-- put your configurations here -->
                </configuration>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-dependency-plugin</artifactId>
                <version>2.10</version>
                <executions>
                    <execution>
                        <id>copy-dependencies</id>
                        <phase>package</phase>
                        <goals>
                            <goal>copy-dependencies</goal>
                        </goals>
                        <configuration>
                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <version>2.15.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>
我爱夜来香A
关注
  • 1
    点赞
  • 踩
  • 2
    收藏
    觉得还不错? 一键收藏
  • 打赏
    打赏
  • 0
    评论
专栏目录
flink多并行数据源下的waterMark触发机制1.pdf
01-22
flink
Flink学习——处理不同数据源的流数据，存入不同的Sink端
Helen_1997_1997的博客
05-17 987
Flink学习——处理不同数据源的流数据，存入不同的Sink端 mysql、kafka、hbase、redis
flink 自定义多并行度的source源
weixin_39150719的博客
09-19 4547
自己定义一个多并行度的source，需要自己实现一个ParallelSourceFunction接口 import org.apache.flink.streaming.api.functions.source.ParallelSourceFunction; /** * interface ParallelSourceFunction<OUT> extends So...
flink 多数据源适配
AinUser的博客
01-21 1971
# 数据支持elasticsearch5/7、clickhouse、cdc github地址：https://github.com/ainusers/flink-adapter-datasource
flink cdc多种数据源安装、配置与验证
eagle89的专栏
09-06 785
flink cdc多种数据源安装、配置与验证
Flink：配置多个数据源和多个数据源并行处理
程序员无羡的博客
04-16 1251
Flink：配置多个数据源和多个数据源并行处理
Flink学习 - 5. Flink之DataSource
jerome520zl的博客
12-23 418
Flink学习 - 5. Flink之DataSourceDataSourceFlink自带的DataSourcesocket给定元素基于集合文件生成自定义DataSourceRichSourceFunction DataSource 简单来说，DataSource是指数据来源，也是其字面意思，Flink无论是实时流计算还是批计算都基于数据进行，其数据源是非常重要的一部分，Flink在命名这一方面...
Flink--source/数据不同的来源
weixin_49180684的博客
08-28 393
文章目录Flink Source不同的来源1、Source几个不同的来源2、kafka作为数据源，flink读取3、myslq数据库作为数据源，且自定义Source Flink Source不同的来源 1、Source几个不同的来源 //创建环境 val env = StreamExecutionEnvironment.getExecutionEnvironment //基本的数据源 //1、端口数据源 env.socketTextStream(“master”，6666) //连接虚拟机等端口，mas
十八、Flink自定义多并行Source
猫猫爱吃小鱼粮
08-13 2589
Flink自定义多并行Source
Flink对两个并行的数据源进行连接操作，如何保证数据的共享？
mcdull213的博客
07-06 8487
当对两个并行的数据源进行连接操作，如何保证数据的共享？ 1. 场景： 在项目中，对两个数据源进行整合，出现了数据丢失的情况。 需求：Kafka数据中domain通过MySQL数据转换为userid Kafka(ip,domain,traffic)=Flink - connect=MySQL(userid,domain)==>Result(ip,userid,traffic） 数据源一...
Flink入门：读取Kafka实时数据流，实现WordCount
01-07
本文主要介绍Flink接收一个Kafka文本数据流，进行WordCount词频统计...设置Kafka相关参数，连接对应的服务器和端口号，读取名为Shakespeare的Topic中的数据源，将数据源命名为stream： // Kafka参数 Properties propert
Flink1.14.4自定义flink-connector-jdbc连接SQLServer和SAP数据库
07-08
Flink1.14.4自定义flink-connector-jdbc连接SQLServer和SAP数据库
Flink Phoenix connector依赖包
06-02
flink sql读写phoenix所使用到的连接器依赖包： flink-sql-connector-phoenix-1.14-1.0.jar 使用示例： create table tab2( ID STRING, NAME STRING, PRIMARY KEY (ID) NOT ENFORCED )WITH( 'connector' = '...
flink-connector-base-1.13.2-API文档-中英对照版.zip
06-05
Maven坐标：org.apache.flink:flink-connector-base:1.13.2； 标签：apache、flink、connector、base、中英对照文档、jar包、java； 使用方法：解压翻译后的API文档，用浏览器打开“index.html”文件，即可纵览文档...
flink-connector-kafka_2.11-1.7.1-API文档-中英对照版.zip
07-14
Maven坐标：org.apache.flink:flink-connector-kafka_2.11:1.7.1； 标签：apache、flink、connector、kafka、中英对照文档、jar包、java； 使用方法：解压翻译后的API文档，用浏览器打开“index.html”文件，即可...
Flink实战5-Flink全局参数传递实现Kafka数据源对接落地Elasticsearch_Sink与过程Search
Kevin__Durant的博客
06-12 1053
背景 适用于配置化传入Flink全局参数； 项目起源于公司多数据源对接，此Demo只是其中一种，整个过程是公司前端部门因为各方渠道传入的不同数据，风控的业务方需要保留进行规则查询；数据流转过程就是Kafka接入不同数据源，按照不同的要求进行es的直接存储或者查询拼接后的存储； 摘要 关键字 Flink_Elasticsearch查询、Elasticsearch_Sink、Flink全局配置化参数； 设计 整个过程比较简单，每个类中注释很详细； Kafka_Source对接数据源； Mapper_P
Flink Broadcast State实用指南
Matrix42
06-27 3505
从1.5.0开始，Flink提供了一种新的State类型，称为Broadcast State。在这篇文章中，我们将解释什么是Broadcast State，并展示如何将其应用于评估事件流上的动态模式的应用的示例。我们将向您介绍处理步骤和源代码，以实现此应用。 什么是Broadcast State？ Broadcast State可用于以特定方式组合和联合处理两个事件流。第一个流的事件被广播到一个算...
Flink进阶教程：如何在两个DataStream上进行Join操作
皮皮鲁的AI星球
01-18 2432
批处理经常要解决的问题是将两个数据源做关联Join操作。比如，很多手机APP都有一个用户数据源User，同时APP会记录用户的行为，我们称之为Behavior，两个表按照userId来进行Join。在流处理场景下，Flink也支持了Join，只不过Flink是在一个时间窗口上来进行两个表的Join。 目前，Flink支持了两种Join：Window Join（窗口连接）和Interval Joi...
flinksql数据源
最新发布
03-30
FlinkSQL中的数据源可以通过使用connector来定义。在给出的引用中，有两个例子可以作为参考。 引用[1]中的例子展示了如何使用`datagen` connector来创建一个临时表，并通过`generateRowUdtf`函数生成新的行。具体...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包

打赏作者

我爱夜来香A

你的鼓励将是我创作的最大动力

¥1 ¥2 ¥4 ¥6 ¥10 ¥20
扫码支付：¥1
获取中
扫码支付

您的余额不足，请更换扫码支付或充值

打赏作者

实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值