java配置hive数据源_java整合hive-jdbc

最新推荐文章于 2024-10-30 22:34:09 发布

weixin_39612110

最新推荐文章于 2024-10-30 22:34:09 发布

阅读量644

点赞数

文章标签： java配置hive数据源

本文链接：https://blog.csdn.net/weixin_39612110/article/details/114542407

版权

importcom.didichuxing.fe.offline.config.HiveConfig;importcom.didichuxing.fe.offline.entity.TableInfo;importcom.didichuxing.fe.offline.util.DateUtil;importcom.didichuxing.fe.offline.util.ParquetShema;importcom.didichuxing.fe.offline.util.SparkTool;importorg.apache.hadoop.conf.Configuration;importorg.apache.spark.sql.Dataset;importorg.apache.spark.sql.Row;importorg.apache.spark.sql.SparkSession;importorg.apache.tomcat.jdbc.pool.DataSource;importorg.slf4j.Logger;importorg.slf4j.LoggerFactory;importorg.springframework.dao.DataAccessException;importorg.springframework.jdbc.core.JdbcTemplate;importorg.apache.hadoop.fs.Path;importjava.nio.file.Paths;importjava.sql.ResultSet;importjava.sql.SQLException;importjava.sql.Statement;importjava.util.ArrayList;importjava.util.List;public classHiveJdbcDao {private static final Logger logger = (Logger) LoggerFactory.getLogger(HiveJdbcDao.class);private static HiveJdbcDao hiveJdbcDao = null;public staticHiveJdbcDao getInstance(){if(hiveJdbcDao == null){synchronized (MysqlBaseDao.class){if (hiveJdbcDao == null){

hiveJdbcDao= newHiveJdbcDao();

}

}returnhiveJdbcDao;

}privateHiveJdbcDao(){

}private DataSource jdbcDataSource =HiveConfig.getInstance().getDataSource();private JdbcTemplate hiveJdbcTemplate =HiveConfig.getInstance().getJdbcTemplate();/*** 查询hive表中字段名以及类型

*@paramabstractSql

*@return*@throwsSQLException*/

public ListselectTableInfoFromHive(String abstractSql){

List tableInfoList = new ArrayList();

TableInfo tableInfo= newTableInfo();

Statement statement= null;

logger.info("Running sql: " +abstractSql);try{

statement=jdbcDataSource.getConnection().createStatement();

ResultSet res=statement.executeQuery(abstractSql);while(res.next()) {

tableInfo.setColumnName(res.getString(1));

tableInfo.setColumnType(res.getString(2));

tableInfo.setColumnComment(res.getString(3));

tableInfoList.add(tableInfo);

}

}catch(SQLException e) {

logger.info(e.getMessage());

}returntableInfoList;

}/*** 查询hive库中表名

*@paramabstractSql

*@return*@throwsSQLException*/

public ListselectTableNameFromHive(String abstractSql){

List tableNameList = new ArrayList();

Statement statement= null;

logger.info("Running sql: " +abstractSql);try{

statement=jdbcDataSource.getConnection().createStatement();

ResultSet res=statement.executeQuery(abstractSql);

logger.error("hive表名String[]: " +res.toString());while(res.next()) {

tableNameList.add(res.getString(1));

}

}catch(SQLException e) {

logger.info(e.getMessage());

}returntableNameList;

}/*** 自动从本地数据加载进入hive

*@paramfilepath*/

public voidloadIntoHiveTable(String filepath, String tableName) {

String dateFileFormat=DateUtil.getYesterdayFileFormat();

String[] dateSplit= dateFileFormat.split("/");

StringBuffer buildSql= newStringBuffer();

buildSql.append("load data inpath " ).append("\'").append(filepath).append("\'")

.append(" into table fe.").append(tableName).append(" partition (year = ")

.append(dateSplit[0]).append(", month = ").append(dateSplit[1])

.append(",day = ").append(dateSplit[2]).append(")");//String sql = "load data inpath " + "\'" + filepath + "\'" +//" into table fe." + tableName + " partition (year = " + dateSplit[0] + ", month = "//+ dateSplit[1] + ",day = " + dateSplit[2] + ")";

logger.info("将数据加载进入hive表的sql : {}", buildSql.toString());try{

hiveJdbcTemplate.execute(buildSql.toString());

}catch(DataAccessException dae) {

logger.error(dae.toString());

}

}/*** 对hive表结构进行更新(增加字段)

*@paramabstractSql*/

public voidupdateHiveTable(String abstractSql) {try{

hiveJdbcTemplate.execute(abstractSql);

}catch(DataAccessException dae) {

logger.error(dae.toString());

}