spark 读取hive表

idea 通过spark sql代码操作hive表
1、依赖pom

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>blibliSpark</artifactId>
        <groupId>com.bupt</groupId>
        <version>1.0-SNAPSHOT</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.bupt</groupId>
    <artifactId>sparkSQL</artifactId>
    <dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>2.1.1</version>
        </dependency>
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.26</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>2.1.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.11</artifactId>
            <version>2.1.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-exec</artifactId>
            <version>1.2.1</version>
        </dependency>

    </dependencies>

</project>

2、拷贝hive-site.xml到resources目录

<configuration  xmlns:xi="http://www.w3.org/2001/XInclude">
    <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://hadoop100:3306/hive?useSSL=false</value>
    </property>

    <property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.jdbc.Driver</value>
    </property>

    <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>root</value>
    </property>

    <property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>123</value>
    </property>

    <property>
        <name>hive.metastore.warehouse.dir</name>
        <value>/user/hive/warehouse</value>
    </property>

    <property>
        <name>hive.metastore.schema.verification</name>
        <value>false</value>
    </property>

    <property>
        <name>hive.exec.scratchdir</name>
        <value>/tmp/spark</value>
    </property>

    <property>
        <name>hive.metastore.client.connect.retry.delay</name>
        <value>5</value>
    </property>

    <property>
        <name>hive.metastore.client.socket.timeout</name>
        <value>1800</value>
    </property>

    <property>
        <name>hive.metastore.uris</name>
        <value>thrift://haoop100:9083</value>
    </property>

    <property>
        <name>hive.server2.enable.doAs</name>
        <value>false</value>
    </property>

    <property>
        <name>hive.server2.thrift.http.port</name>
        <value>10002</value>
    </property>

    <property>
        <name>hive.server2.thrift.port</name>
        <value>10016</value>
    </property>

    <property>
        <name>hive.server2.transport.mode</name>
        <value>binary</value>
    </property>

    <property>
        <name>metastore.catalog.default</name>
        <value>spark</value>
    </property>
</configuration>

3 代码实现

package day09

import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession

/**
 * @author yangkun
 * @date 2020/10/29 20:54
 * @version 1.0
 */
object Spark02_Hive {
  def main(args: Array[String]): Unit = {
    //创建配置文件对象
    val conf: SparkConf = new SparkConf().setMaster("local[*]").setAppName("SparkSQL02_MySQL")
    //创建SparkSession对象
    val spark: SparkSession = SparkSession.builder().enableHiveSupport().config(conf).getOrCreate()

    spark.sql("show tables").show()

    //释放资源
    spark.stop()
  }
}