在Spring中配置Spark hive-thriftserver的连接DataSource与配置其他数据源连接方式是一样的,如一般Oracle数据源配置,使用如下必须的Jar包:
使用JDBC程序示例:
package com.hadoop.test;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
public class ThreadSql extends Thread {
@Override
public void run() {
excSparkSql();
}
public void excSparkSql() {
String result = "";
Connection conn = null;
Statement stmt = null;
try {
Class.forName("org.apache.hadoop.hive.jdbc.HiveDriver");
conn = DriverManager.getConnection(
"jdbc:hive://10.20.12.214:10000/test_db", "hduser001", "");
stmt = conn.createStatement();
String sql = " select count(1) from test_data ";
long start = System.currentTimeMillis();
ResultSet res = stmt.executeQuery(sql);
long cost = System.currentTimeMillis() - start;
result = Thread.currentThread().getName() + ": " + cost/1000.0f + "s";
System.out.println(result);
stmt.close();
conn.close();
} catch (Exception e) {
try {
stmt.close();
conn.close();
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
e.printStackTrace();
}
}
}
在Spring中配置如下,类似oracle的datasource配置,并在其他的ServiceBean中将其注入即可
<bean id="dataService" class="com.sun.dt.service.impl.DataServiceImpl"> <property name="dataDao" ref="sparkDataSource"></property> </bean> <bean id="oralceDataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close"> <property name="driverClassName" value="${jdbc.driver}" /> <property name="url" value="${jdbc.url}" /> <property name="username" value="${jdbc.username}" /> <property name="password" value="${jdbc.password}" /> <property name="maxActive" value="${jdbc.maxActive}" /> <property name="maxIdle" value="${jdbc.maxIdle}" /> <property name="maxWait" value="${jdbc.maxWait}" /> <property name="defaultAutoCommit" value="${jdbc.defaultAutoCommit}" /> </bean> <bean id="sparkDataSource" class="org.apache.commons.dbcp.BasicDataSource" destroy-method="close"> <property name="driverClassName" value="org.apache.hadoop.hive.jdbc.HiveDriver" /> <property name="url" value="jdbc:hive://192.168.10.124:10000/test_db" /> <property name="username" value="hduser001" /> <property name="password" value="" /> <property name="maxActive" value="8" /> <property name="maxIdle" value="3" /> <property name="maxWait" value="5" /> </bean>
转载于:https://blog.51cto.com/xubcing/1607384