mysql并行读_并行读取MySQL数据

...

private val dbUrl =

s"""jdbc:mysql://${host}:${port}/${db_name}

|?zeroDateTimeBehavior=convertToNull

|&read_buffer_size=100M""".stripMargin.replace("

", "")

def run(sqlQuery: String): DataFrame = {

println(sqlQuery)

Datapipeline.spark.read

.format("jdbc")

.option("driver", "com.mysql.jdbc.Driver")

.option("url", dbUrl)

.option("user", user)

.option("password", pass)

.option("dbtable", s"($sqlQuery) as tmp")

.load()

}

...

def getBounds(table: String, whereClause: String, partitionColumn: String): Array[Int] = {

val sql = s"select min($partitionColumn) as min, max($partitionColumn) as max from $table${

if (whereClau