SpringBoot DataSource集成Hive、Phoenix、Presto数据源(带Kerberos校验)

最新推荐文章于 2024-09-12 18:00:00 发布

林的BLOG

最新推荐文章于 2024-09-12 18:00:00 发布

阅读量1.3k

点赞数

文章标签：大数据 kerberos impala hive hadoop

本文链接：https://blog.csdn.net/jayjwes190/article/details/108709421

版权

在数据中台的落地过程中，元数据管理经常需要和多个数据源进行交互，下面我们延时几个数据源

1. HiveDataSource

@Configuration
@EnableTransactionManagement
public class HiveDataSource {

    @Value("${spring.datasource.hive.url}")
    private String url;

    @Value("${project.release.principal}")
    private String principal;

    @Value("${project.release.keytab}")
    private String keytab;

    @Bean(name = "hiveJdbcDataSource")
    @Qualifier("hiveJdbcDataSource")
    public DataSource dataSource() throws IOException {
        DruidDataSource dataSource = new DruidDataSource();
        dataSource.setUrl(url);
        org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
        conf.set("hadoop.security.authentication", "Kerberos");
        UserGroupInformation.setConfiguration(conf);
        UserGroupInformation.loginUserFromKeytab(principal, keytab);
        return dataSource;
    }

    @Bean(name = "hiveJdbcTemplate")
    public JdbcTemplate hiveJdbcTemplate(@Qualifier("hiveJdbcDataSource") DataSource dataSource) {
        return new JdbcTemplate(dataSource);
    }

}

2.PhoenixDataSource

@Configuration
@EnableTransactionManagement
public class PhoenixDataSource {

    @Value("${spring.datasource.phoenix.url}")
    private String url;

    @Value("${project.release.principal}")
    private String principal;

    @Value("${project.release.keytab}")
    private String keytab;

    @Bean(name = "phoenixJdbcDataSource")
    @Qualifier("phoenixJdbcDataSource")
    public DataSource dataSource() {
        DruidDataSource dataSource = new DruidDataSource();
        dataSource.setUrl(url);
        dataSource.setDriverClassName("org.apache.phoenix.queryserver.client.Driver");
        Properties properties = new Properties();
        properties.setProperty("serialization", "PROTOBUF");
        properties.setProperty("authentication", "SPNEGO");
        properties.setProperty("principal", principal);
        properties.setProperty("keytab", keytab);
        dataSource.setConnectProperties(properties);
        return dataSource;
    }

    @Bean(name = "phoenixJdbcTemplate")
    public JdbcTemplate phoenixJdbcTemplate(@Qualifier("phoenixJdbcDataSource") DataSource dataSource) {
        return new JdbcTemplate(dataSource);
    }

}

3.PrestoDataSource

@Configuration
@EnableTransactionManagement
public class PrestoDataSource {

    @Value("${spring.datasource.presto.url}")
    private String url;

    @Value("${project.release.principal}")
    private String principal;

    @Value("${project.release.keytab}")
    private String keytab;

    @Bean(name = "prestoJdbcDataSource")
    @Qualifier("prestoJdbcDataSource")
    public DataSource dataSource() {
        DruidDataSource dataSource = new DruidDataSource();
        dataSource.setUrl(url);
        String driver = "com.facebook.presto.jdbc.PrestoDriver";
        dataSource.setDriverClassName(driver);
        dataSource.setUsername("wumii-data");
//        Properties properties=new Properties();
//        properties.setProperty("KerberosPrincipal",principal);
//        properties.setProperty("KerberosKeytabPath",keytab);
//        dataSource.setConnectProperties(properties);
        return dataSource;
    }

    @Bean(name = "prestoJdbcTemplate")
    public JdbcTemplate phoenixJdbcTemplate(@Qualifier("prestoJdbcDataSource") DataSource dataSource) {
        return new JdbcTemplate(dataSource);
    }

}

涉及到的Properties

spring.datasource.hive.url=jdbc:hive2://host1:2181,host2.wumii.net:2181,host3:2181/default;principal=hive/_HOST@WUMII.NET;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2-interactive
spring.datasource.phoenix.url=jdbc:phoenix:thin:url=http://host3:8765
spring.datasource.presto.url=jdbc:presto://hive@host2:8999/hive/default
project.release.keytab=XXX.keytab
project.release.principal=XX@WUMII.NET