flink scala mysql_Flink 读取 MySQL 示例

最新推荐文章于 2024-10-28 16:53:58 发布

射命丸咲

最新推荐文章于 2024-10-28 16:53:58 发布

阅读量789

点赞数

文章标签： flink scala mysql

本文链接：https://blog.csdn.net/weixin_34012046/article/details/113565517

版权

本文展示了如何使用 Flink Scala API 从 MySQL 数据库读取数据。提供了三种不同的方法创建 JDBC 输入格式，查询 MySQL 数据并打印输出，最后执行 Flink 作业。

摘要由CSDN通过智能技术生成

package org.onepiece.bigdata.windows.JDBCimportorg.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}importorg.apache.flink.api.java.io.jdbc.{JDBCAppendTableSink, JDBCInputFormat, JDBCOutputFormat}importorg.apache.flink.table.api.scala.BatchTableEnvironmentimportorg.apache.flink.table.api.{DataTypes, EnvironmentSettings, Types}importorg.apache.flink.types.Rowimportorg.apache.flink.api.java.typeutils.RowTypeInfoimportorg.apache.flink.streaming.api.scala.StreamExecutionEnvironmentimportorg.apache.flink.table.api.scala.StreamTableEnvironmentimportorg.apache.flink.table.api.Table

object mysql_test {importorg.apache.flink.api.scala.extensions._importorg.apache.flink.api.scala._

protected val host_name= "localhost"protected val port= 3306protected val db_name= "vn09jj5"protected val url= s"jdbc:mysql://${host_name}:${port}/${db_name}?useSSL=false&serverTimezone=UTC"val url_flink= "jdbc:mysql://localhost:3306/flink_db?useSSL=false&serverTimezone=UTC"protected val driver= "com.mysql.cj.jdbc.Driver"protected val user= "root"protected val password= "sa123ADMIN"

def getBatchExecutionEnvironment(): ExecutionEnvironment ={

val benv=ExecutionEnvironment.getExecutionEnvironmentreturnbenv

}def getStreamExecutionEnvironment(): StreamExecutionEnvironment ={

val senv=StreamExecutionEnvironment.getExecutionEnvironmentreturnsenv

}def getEnvironmentSettings(): EnvironmentSettings ={

val setting=EnvironmentSettings.newInstance()

.useBlinkPlanner()

.inStreamingMode()

.build()returnsetting

}def mysql_read_1(): Unit ={

val benv=ExecutionEnvironment.getExecutionEnvironment//读取MySQL

val dataSource=benv.createInput(

JDBCInputFormat.buildJDBCInputFormat()

.setDBUrl(url)

.setDrivername(driver)

.setUsername(user)

.setPassword(password)

.setQuery("select phone_Nbr,channel from cte_phone_channel order by phone_Nbr")

.setRowTypeInfo(new RowTypeInfo(

BasicTypeInfo.STRING_TYPE_INFO,//phone_Nbr

BasicTypeInfo.STRING_TYPE_INFO//channel

))

.finish()

)

dataSource.print()

println(dataSource.count())

benv.execute("mysql-test")

}def mysql_read_2(): Unit ={

val benv=ExecutionEnvironment.getExecutionEnvironment

val query_table=

"""|select a.phone_Nbr,a.channel,b.isDelete,b.remart

|from cte_phone_channel as a

|inner join order_phone as b on b.phone_Nbr=a.phone_Nbr and b.isDelete=1""".stripMargin//读取MySQL

val dataSource=benv.createInput(

JDBCInputFormat.buildJDBCInputFormat()

.setDBUrl(url)

.setDrivername(driver)

.setUsername(user)

.setPassword(password)

.setQuery(query_table)

.setRowTypeInfo(new RowTypeInfo(

BasicTypeInfo.STRING_TYPE_INFO,//phone_Nbr

BasicTypeInfo.STRING_TYPE_INFO,//channel

BasicTypeInfo.INT_TYPE_INFO,//isDelete

BasicTypeInfo.STRING_TYPE_INFO//remart

))

.finish()

)

dataSource.print()

println(dataSource.count())

benv.execute("mysql-test")

}def mysql_read_3(): Unit ={

val benv=ExecutionEnvironment.getExecutionEnvironment

val query= "select phone_Nbr,channel from cte_phone_channel order by phone_Nbr"val fields= Array[String]("phone_Nbr", "channel")

val types=Array[TypeInformation[_]](Types.STRING, Types.STRING)

val typeInfo=new RowTypeInfo(types, fields)//读取MySQL

val dataSource=benv.createInput(

JDBCInputFormat.buildJDBCInputFormat()

.setDBUrl(url)

.setDrivername(driver)

.setUsername(user)

.setPassword(password)

.setQuery(query)

.setRowTypeInfo(typeInfo)

.finish()

)

dataSource.print()

println(dataSource.count())

benv.execute("mysql-test")

}