import com.rmd.machinelearn.etl.EtlDataSql
import com.rmd.machinelearn.tools.{DateUtils, HiveUtil, RedisClusterClient}
import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects}
import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spark.{SparkConf, SparkContext}
import redis.clients.jedis.params.ScanParams
import java.sql.{Connection, Statement}
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import java.time.temporal.TemporalAdjusters
import java.time.temporal.WeekFields
import java.util.Locale
import java.util.stream.IntStream
import scala.io.Source
import scala.jdk.CollectionConverters._
object a {
object HiveSqlDialect extends JdbcDialect {
//
override def canHandle(url: String): Boolean = {
url.toLowerCase(Locale.ROOT).startsWith("jdbc:hive2")
// url.startsWith("jdbc:hive2")
//mysql url.toLowerCase(Locale.ROOT).startsWith("jdbc:mysql")
}
override def quoteIdentifier(colName: String): String = {
colName.split('.').map(part => s"`$part`").mkString(".")
// mysql s"`$colName`"
}
}
object RegisterHiveSqlDialect {
def register(): Unit = {
JdbcDialects.registerDialect(HiveSqlDialect)
}
}
def main(args: Array[String]): Unit = {
val spark = SparkSession.builder()
.appName("Hive Example")
.master("local[*]")
.enableHiveSupport()
.getOrCreate()
import spark.implicits._
RegisterHiveSqlDialect.register()
val connectionProperties = new java.util.Properties()
connectionProperties.put("user", "hadoop")
connectionProperties.put("password", "hadoop")
connectionProperties.put("driver", "org.apache.hive.jdbc.HiveDriver")
val query = "(SELECT vodname, vodid FROM vodid) AS voddata"
val jdbcDF: DataFrame = spark.read.jdbc("jdbc:hive2://0.0.0.0:10000/test", query, connectionProperties)
jdbcDF.show()
//
// // 关闭SparkSession
// spark.stop()
// var stmt: Statement = null
// var conn: Connection = null
// try {
// conn = HiveUtil.getInstance().getConnection
// stmt = conn.createStatement()
// val rs = stmt.executeQuery("SELECT vodname, vodid FROM test.vodid")
// while (rs.next()){
// println(rs.getString(1))
// }
// } catch {
// case e: Exception => e.printStackTrace()
// } finally {
// if (null != stmt) {
// stmt.close()
// }
// if (null != conn) {
// conn.close()
// }
// }
}
}
处理spark查询hive时返回的结构全是列名的问题
于 2024-03-20 15:28:13 首次发布