SpringBoot 基于Aspect实现多数据源，支持hive数据源（一）

最新推荐文章于 2023-04-28 15:00:28 发布

hnmpf

最新推荐文章于 2023-04-28 15:00:28 发布

阅读量650

点赞数

分类专栏：大数据

本文链接：https://blog.csdn.net/hnmpf/article/details/103119348

版权

大数据专栏收录该内容

6 篇文章 2 订阅

订阅专栏

项目背景：

最近在摸索中接触大数据展示项目，提到大数据统计就不得不解决多数据源，动态数据源问题，当然这些问题可以通过其他方案替代，但响应的需要付出一定的代价。本笔记中主要探讨的是在一个简单的springboot 项目中引入多种数据源，用户根据业务自动切换响应的数据源进行业务统计，或者在一个API中实现多数据源联合统计。

工具：

STS

施工：

新建项目工程:

maven 配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<parent>
		<groupId>org.springframework.boot</groupId>
		<artifactId>spring-boot-starter-parent</artifactId>
		<version>2.2.1.RELEASE</version>
		<relativePath/> <!-- lookup parent from repository -->
	</parent>
	<groupId>com.example</groupId>
	<artifactId>dataSource</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<name>dataSource</name>
	<description>Demo project for Spring Boot</description>

	<properties>
		<java.version>1.8</java.version>
	</properties>

	<dependencies>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-jdbc</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-jpa</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-data-rest</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-jdbc</artifactId>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-web</artifactId>
		</dependency>
		<dependency>
			<groupId>org.mybatis.spring.boot</groupId>
			<artifactId>mybatis-spring-boot-starter</artifactId>
			<version>2.1.1</version>
		</dependency>

		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-devtools</artifactId>
			<scope>runtime</scope>
			<optional>true</optional>
		</dependency>
		<dependency>
			<groupId>com.h2database</groupId>
			<artifactId>h2</artifactId>
			<scope>runtime</scope>
		</dependency>
		<dependency>
			<groupId>mysql</groupId>
			<artifactId>mysql-connector-java</artifactId>
			<scope>runtime</scope>
		</dependency>
		<dependency>
			<groupId>org.springframework.boot</groupId>
			<artifactId>spring-boot-starter-test</artifactId>
			<scope>test</scope>
			<exclusions>
				<exclusion>
					<groupId>org.junit.vintage</groupId>
					<artifactId>junit-vintage-engine</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.springframework.data</groupId>
			<artifactId>spring-data-hadoop</artifactId>
			<version>2.5.0.RELEASE</version>
		</dependency>
        <!--引入hive依赖和jdbc驱动-->
        <dependency>
			<groupId>org.apache.hive</groupId>
			<artifactId>hive-jdbc</artifactId>
			<version>2.3.3</version>
			<exclusions>
				<exclusion>
					<groupId>org.eclipse.jetty.aggregate</groupId>
					<artifactId>*</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
        <!--为了增强mybatis的部分能力，我们引入mybatis-plus增强框架-->
		<dependency>
		    <groupId>com.baomidou</groupId>
		    <artifactId>mybatis-plus-boot-starter</artifactId>
		    <version>2.2.0</version>
		</dependency>
		<dependency>
			<groupId>com.baomidou</groupId>
			<artifactId>mybatis-plus</artifactId>
			<version>2.3</version>
		</dependency>

	</dependencies>

	<build>
		<plugins>
			<plugin>
				<groupId>org.springframework.boot</groupId>
				<artifactId>spring-boot-maven-plugin</artifactId>
			</plugin>
		</plugins>
	</build>

</project>

数据源配置（application.properties）


spring.datasource.primary.jdbc-url=jdbc:mysql://ip:3306/XXX?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT&useSSL=false
spring.datasource.primary.username=XXX
spring.datasource.primary.password=XXX
spring.datasource.primary.driver-class=com.mysql.cj.jdbc.Driver

spring.datasource.secondary.jdbc-url=jdbc:mysql://xxx:3306/XXX?useUnicode=true&characterEncoding=utf-8&serverTimezone=GMT&useSSL=false
spring.datasource.secondary.username=XXX
spring.datasource.secondary.password=XXX
spring.datasource.secondary.driver-class=com.mysql.cj.jdbc.Driver

spring.datasource.third.jdbc-url=jdbc:hive2://XXX:10000/default
spring.datasource.third.username=XXX
spring.datasource.third.password=XXX
spring.datasource.third.driver-class=org.apache.hive.jdbc.HiveDriver
spring.datasource.third.type=com.zaxxer.hikari.HikariDataSource

动态数据源

package com.example.demo.config;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import javax.sql.DataSource;

import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import org.springframework.transaction.PlatformTransactionManager;

import com.baomidou.mybatisplus.spring.MybatisSqlSessionFactoryBean;

@Configuration
@MapperScan(basePackages = "com.example.demo.dao.mapper")
public class DataSourceConfig {
	
	@Bean("mysqlDataSource")
    @Primary
    @ConfigurationProperties(prefix = "spring.datasource.primary")
    public DataSource mysqlDataSource() {
        DataSource build = DataSourceBuilder.create().build();
        return build;
    }

    @Bean("oracleDataSource")
    @ConfigurationProperties(prefix = "spring.datasource.secondary")
    public DataSource oracleDataSource(){
        return DataSourceBuilder.create().build();
    }
    
    @Bean("hiveDataSource")
    @ConfigurationProperties(prefix = "spring.datasource.third")
    public DataSource hiveDataSource(){
        return DataSourceBuilder.create().build();
    }
    
    @Bean
    public DynamicDataSource dataSource(@Qualifier("mysqlDataSource") DataSource mysqlDataSource,
                                        @Qualifier("oracleDataSource") DataSource oracleDataSource,
                                        @Qualifier("hiveDataSource") DataSource hiveDataSource) {
        Map<Object, Object> map = new HashMap<>();
        map.put(DataSourceType.mysql, mysqlDataSource);
        map.put(DataSourceType.oracle, oracleDataSource);
        map.put(DataSourceType.hive, hiveDataSource);

        DynamicDataSource dynamicDataSource = new DynamicDataSource();
        dynamicDataSource.setTargetDataSources(map);
        dynamicDataSource.setDefaultTargetDataSource(mysqlDataSource);

        return dynamicDataSource;
    }
    
//    @Bean
//    public SqlSessionFactory sqlSessionFactory(DynamicDataSource dynamicDataSource) throws Exception {
//        SqlSessionFactoryBean factoryBean = new SqlSessionFactoryBean();
//        factoryBean.setDataSource(dynamicDataSource);
//        Resource[] resources = new PathMatchingResourcePatternResolver().getResources("classpath*:mapping/*.xml");
//        factoryBean.setMapperLocations(resources);
//        return factoryBean.getObject();
//    }
    //引入mybatis_plus增强包后必须通过MybatisSqlSessionFactoryBean进行session工厂建设，SqlSessionFactory不能满足，此处配置错误会报mapper找不到或无法实例化问题。
    @Bean
    public MybatisSqlSessionFactoryBean mybatisSqlSessionFactoryBean(DynamicDataSource dynamicDataSource) throws Exception {
        MybatisSqlSessionFactoryBean mybatisPlus = new MybatisSqlSessionFactoryBean();
        mybatisPlus.setDataSource(dynamicDataSource);
        Resource[] resources = new PathMatchingResourcePatternResolver().getResources("classpath*:mapping/*.xml");
        mybatisPlus.setMapperLocations(resources);
        return mybatisPlus;
    }

    
    @Bean
    public PlatformTransactionManager transactionManager(DynamicDataSource dynamicDataSource){
        return new DataSourceTransactionManager(dynamicDataSource);
    }

}

数据源枚举

package com.example.demo.config;

public enum DataSourceType {
	mysql,
    oracle,
    hive
}

数据源注解

package com.example.demo.config;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

@Target({ElementType.METHOD, ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface DataSource {
    DataSourceType value() default DataSourceType.mysql;
}

数据源切面

package com.example.demo.config;


import org.aspectj.lang.annotation.After;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;


@Aspect
// 通过order指定数据源加载顺序一定是：先加载数据源再加载事务，否则会遇到事务丢失问题
@Order(-10)
@Component
public class DataSourceAspect {

        // 数据源切入点，切点就是注解DataSource，通过@annotation指定该切点的有效范围
	    @Before("@annotation(ds)")
	    public void beforeDataSource(DataSource ds) {
	        DataSourceType value = ds.value();
	        DataSourceContextHolder.setDataSource(value);
	        
	    }
	    @After("@annotation(ds)")
	    public void afterDataSource(DataSource ds){
	        DataSourceContextHolder.clearDataSource();
	    }
}

动态数据源主类

package com.example.demo.config;

import org.springframework.jdbc.datasource.lookup.AbstractRoutingDataSource;

public class DynamicDataSource extends AbstractRoutingDataSource {

	@Override
	protected Object determineCurrentLookupKey() {
		// TODO Auto-generated method stub
		return DataSourceContextHolder.getDataSource();
	}

}

UserInfoModel （PO）

package com.example.demo.entry;

import com.baomidou.mybatisplus.annotations.TableField;
import com.baomidou.mybatisplus.annotations.TableId;
import com.baomidou.mybatisplus.annotations.TableName;
import com.baomidou.mybatisplus.enums.IdType;

@TableName(value="T_USER")
public class UserInfoModel {
	
	@TableId(value = "id",type = IdType.AUTO)
	private String id;
	
	@TableField(value = "name",exist = true)
	private String userName;
	
	@TableField(value = "password",exist = true)
	private String passWord;

	public String getUserName() {
		return userName;
	}

	public void setUserName(String userName) {
		this.userName = userName;
	}

	public String getPassWord() {
		return passWord;
	}

	public void setPassWord(String passWord) {
		this.passWord = passWord;
	}

}

DataSourceContextHolder

package com.example.demo.config;

public class DataSourceContextHolder {

	// 存放当前线程使用的数据源类型
    private static final ThreadLocal<DataSourceType> contextHolder = new ThreadLocal<>();
    
 // 设置数据源
    public static void setDataSource(DataSourceType type){
        contextHolder.set(type);
    }

    // 获取数据源
    public static DataSourceType getDataSource(){
        return contextHolder.get();
    }

    // 清除数据源
    public static void clearDataSource(){
        contextHolder.remove();
    }
}

UserInfoMapper（DAO）

package com.example.demo.dao.mapper;

import java.util.List;

import org.apache.ibatis.annotations.Select;
import org.springframework.stereotype.Repository;

import com.baomidou.mybatisplus.mapper.BaseMapper;
import com.example.demo.config.DataSource;
import com.example.demo.config.DataSourceType;
import com.example.demo.entry.UserInfoModel;

@Repository
public interface UserInfoMapper extends BaseMapper<UserInfoModel>{
    //dao 层切换数据源
	@Select(" select * from t_user ")
	@DataSource(value=DataSourceType.mysql)
    List<UserInfoModel> findAll();
	
	@Select(" select * from t_user ")
	@DataSource(value=DataSourceType.oracle)
    List<UserInfoModel> findAll2();
	
	@Select(" select t.name as userName,t.hobby as passWord from default.mytable1 t ")
	@DataSource(value=DataSourceType.hive)
    List<UserInfoModel> findAll3();
	
}

UserInfoServerImp（SERVER）

package com.example.demo.servers.imp;

import java.util.List;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import com.example.demo.config.DataSource;
import com.example.demo.config.DataSourceType;
import com.example.demo.dao.mapper.UserInfoMapper;
import com.example.demo.entry.UserInfoModel;
import com.example.demo.servers.IUserInfoServer;

@Service
public class UserInfoServerImp implements IUserInfoServer {
	
	@Autowired
	private UserInfoMapper usermapper;

	@Override
	public List<UserInfoModel> findAll() {
		// TODO Auto-generated method stub
		return usermapper.findAll();
	}

	@Override
	public List<UserInfoModel> findAll2() {
		// TODO Auto-generated method stub
		return usermapper.findAll2();
	}
	
	@Override
	public List<UserInfoModel> findAll3() {
		// TODO Auto-generated method stub
		List<UserInfoModel> list = usermapper.findAll3();
		return list;
	}
	
	@Override
	public List<UserInfoModel> findAll4() {
		// server 方法中调用dao 层不同数据源的数据，findAll3是hive数据源,findAll2是oracle数据源
		List<UserInfoModel> list = usermapper.findAll3();
		list.addAll(usermapper.findAll2());
		return list;
	}
	// server层方法中切换数据源.
	@DataSource(value=DataSourceType.mysql)
	public void insertDb(UserInfoModel model) {
		usermapper.insert(model);
	}
	
	@DataSource(value=DataSourceType.oracle)
	public void insertDb2(UserInfoModel model) {
		usermapper.insert(model);
	}

}

UserInfoControl (C)

package com.example.demo.control;

import java.util.List;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import com.example.demo.entry.UserInfoModel;
import com.example.demo.servers.imp.UserInfoServerImp;

@RestController
public class UserInfoControl {

	@Autowired
	private UserInfoServerImp userInfoServer;
	
	@RequestMapping("selectuser")
	public List<UserInfoModel> findAll(){
		return userInfoServer.findAll();
		
	} 
	
	@RequestMapping("selectuser2")
	public List<UserInfoModel> findAll2(){
		return userInfoServer.findAll2();
		
	} 
	
	@RequestMapping("selectuser3")
	public List<UserInfoModel> findAll3(){
		return userInfoServer.findAll3();
		
	} 
	
	@RequestMapping("selectuser4")
	public List<UserInfoModel> findAll4(){
		return userInfoServer.findAll4();
		
	} 
	
	@RequestMapping("insert")
	public void insert(){
		UserInfoModel model = new UserInfoModel();
		model.setPassWord("1111111");
		model.setUserName("mpftest");
		userInfoServer.insertDb(model);
		
	}
	
	@RequestMapping("insert2")
	public void insert2(){
		UserInfoModel model = new UserInfoModel();
		model.setPassWord("1111111");
		model.setUserName("mpftest");
		userInfoServer.insertDb2(model);
		userInfoServer.insertDb(model);
		
	}
}

总结

通过上面完整的例子，我们已经建立了一个支持3种数据源的springboot项目模型，能够满足自定义的SQL在server层和DAO层通过注解切换数据源，但针对mybatis_plus的增强API支持在server和dao层切换数据源进行增删改查，但需要注意，在服务层（server）不能实现多个不同数据源的接口在一个方法中调用，若有这个需求，可以考虑把整合多数据源统计的方法写在control层。

亮点：注解实现数据源切换可以实现在同一个server或dao层的不同方法使用不同的注解。

通过后续的改造可以实现真正的动态数据源切换。

下篇预告

SpringBoot 基于分包分路径的方式实现多数据源，支持hive数据源（二）

SpringBoot基于Aspect实现动态数据源切换，支持hive数据源（三）

SpringBoot实现与HBase 数据库通过Phoenix 实现集成（四）

hnmpf

关注

0
点赞
踩
4

收藏

觉得还不错? 一键收藏
0
评论
SpringBoot 基于Aspect实现多数据源，支持hive数据源（一）

项目背景：最近在摸索中接触大数据展示项目，提到大数据统计就不得不解决多数据源，动态数据源问题，当然这些问题可以通过其他方案替代，但响应的需要付出一定的代价。本笔记中主要探讨的是在一个简单的springboot 项目中引入多种数据源，用户根据业务自动切换响应的数据源进行业务统计，或者在一个API中实现多数据源联合统计。工具： STS施工：新建项目工程:...
复制链接

扫一扫

专栏目录