连接hive的方法有很多,这里说说通过jdbc连接hive
首先要把hive下的conf下的hive-site.xml文件复制到编译器中
object readAndWriteHive {
//从hive中读取数据表,去重并写到一个新表中
def main(args: Array[String]): Unit = {
//加载配置文件
val load = ConfigFactory.load()
//加载驱动
Class.forName(load.getString("hivedriverClassName")).newInstance()
//建立jdbc连接
val conn = DriverManager.getConnection(load.getString("hiveurl"),"hive","hive")
//sql语句1,从表中查询数据并去重
val statement = conn.prepareStatement("select distinct * from stock")
//sql语句2,将数据写到一个新表中
val statement2 = conn.prepareStatement("insert into distinct_stock2 values(?,?,?,?,?,?,?,?,?)")
//执行sql语句1
val resultSet = statement.executeQuery()
while (resultSet.next()){
//获取表中数据的字段信息
val exchangename = resultSet.getString("exchangename")
val code = resultSet.getString("code")
val transdate = resultSet.getString("transdate")
val transtime = resultSet.getString("transtime")
val price = resultSet.getString("price")
val pricefluncuation = resultSet.getString("pricefluncuation")
val volume = resultSet.getString("volume")
val turnover = resultSet.getString("turnover")
val nature = resultSet.getString("nature")
//将字段插入新表中
statement2.setString(1,exchangename)
statement2.setString(2,code)
statement2.setString(3,transdate)
statement2.setString(4,transtime)
statement2.setString(5,price)
statement2.setString(6,pricefluncuation)
statement2.setString(7,volume)
statement2.setString(8,turnover)
statement2.setString(9,nature)
//执行sql语句2
statement2.execute()
}
println("完成")
statement.close()
statement2.close()
conn.close()
}
}