Springbatch读取rabbitmq队列数据 持久化mongoDB以及使用mybatis持久化至mysql

于 2019-03-19 16:53:05 发布
阅读量2.2k 收藏 5
点赞数 3
分类专栏： springbatch rabbitmq springboot 文章标签： springbatch rabbitmq mysql mongodb springboot
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/warmer_winter/article/details/88579676
版权

史上最简单最方便的Springbatch读取rabbitmq队列数据持久化至mysql以及mongoDB，当然我们还是以springboot为项目框架。

我采用的通用设计：通用reader以及通用writer，设计这样目的是为了使用更加方便。

第一步我们先导入maven映射：

<!-- rabbitmq 映射 -->
<dependency>
	<groupId>org.springframework.cloud</groupId>
	<artifactId>spring-cloud-starter-stream-rabbit</artifactId>
</dependency>
<dependency>
    <groupId>org.springframework.boot</groupId>
	<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
<dependency>
	<groupId>org.springframework.cloud</groupId>
	<artifactId>spring-cloud-starter-bus-amqp</artifactId>
</dependency>

<!-- mongodb 映射 -->
<dependency>
	<groupId>org.springframework.boot</groupId>
	<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>

<!-- springbatch 映射 -->
<dependency>
	<groupId>org.springframework.boot</groupId>
	<artifactId>spring-boot-starter-batch</artifactId>
</dependency>

<!-- mysql 连接池 -->
<dependency>
	<groupId>mysql</groupId>
	<artifactId>mysql-connector-java</artifactId>
	<version>6.0.6</version>
</dependency>
<dependency>
	<groupId>com.alibaba</groupId>
	<artifactId>druid</artifactId>
	<version>1.1.2</version>
</dependency>

 第二步我们配置rabbitmq、mongodb、mysql链接信息以及mybatis

spring.application.name=test-batch
spring.batch.job.enabled=false
server.port=8088

spring.rabbitmq.host=10.10.50.xxx
spring.rabbitmq.port=5672
spring.rabbitmq.username=root
spring.rabbitmq.password=root

spring.datasource.url=jdbc:mysql://10.10.50.xxx:3306/test_batch?Unicode=true&characterEncoding=UTF-8
spring.datasource.type=com.alibaba.druid.pool.DruidDataSource
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.username=test
spring.datasource.password=test
spring.datasource.initialSize=5
spring.datasource.minIdle=5
spring.datasource.maxActive=20
spring.datasource.maxWait=60000
spring.datasource.timeBetweenEvictionRunsMillis=3600000
spring.datasource.minEvictableIdleTimeMillis=18000000
spring.datasource.validationQuery=SELECT 1 FROM DUAL
spring.datasource.testWhileIdle=true
spring.datasource.testOnBorrow=true
spring.datasource.testOnReturn=true
spring.datasource.poolPreparedStatements=true
spring.datasource.maxPoolPreparedStatementPerConnectionSize=20
spring.datasource.filters=stat,wall,log4j
spring.datasource.connectionProperties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000
jpa.hibernate.ddl-auto.show-sql= true

mongo.applicationName= mongoserver
mongo.description = mongo server module
mongo.connectionsPerHost= 10  
mongo.minConnectionsPerHost= 1  
mongo.threadsAllowedToBlockForConnectionMultiplier= 5 
mongo.cursorFinalizerEnabled= true 
mongo.maxWaitTime= 120000
mongo.connectTimeout= 100000 
mongo.socketTimeout= 30000
mongo.socketKeepAlive= true 
mongo.maxConnectionIdleTime= 60000
mongo.maxConnectionLifeTime= 0 
mongo.serverSelectionTimeout= 30 
mongo.heartbeatSocketTimeout= 1000 
mongo.heartbeatConnectTimeout= 1500 
mongo.minHeartbeatFrequency= 5
mongo.heartbeatFrequency= 10
mongo.alwaysUseMBeans= true 
mongo.uri= mongodb://user:pass@10.10.50.xxx:27001/test_batch

mybatis.config-location=classpath:mybatis-config.xml
mybatis.mapper-locations=classpath:/com/xx/xx/batch/mapper/*.xml

第三步配置MongoDB连接池

import com.mongodb.MongoClient;
import com.mongodb.MongoClientOptions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.mongo.MongoProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import org.springframework.core.env.Environment;
import org.springframework.data.mongodb.core.MongoTemplate;

import java.net.UnknownHostException;


@Configuration
@EnableConfigurationProperties(MongoProperties.class)
public class MongodbConfig {
	Logger logger = LoggerFactory.getLogger(MongodbConfig.class);

	@Autowired
	private MongoProperties properties;

	@Autowired
	private MongodbProperties mongodbProperties;

	@Autowired(required = false)
	private MongoClientOptions options;
	@Autowired
	private Environment env;

	private MongoClient mongo;


	@Bean
	@Autowired
	public MongoTemplate mongoTemplate(MongoClient mongo) {
		int index = mongodbProperties.getUri().lastIndexOf('/');
		String database = mongodbProperties.getUri().substring(++index);
		MongoTemplate mongoTemplate = new MongoTemplate(mongo, database);
		return mongoTemplate;
	}

	@Bean
	@Primary // 该实例优先与其他实例注入
	public MongoClient mongo() throws UnknownHostException {
		mongodbProperties.getApplicationName();
		this.options = MongoClientOptions.builder().applicationName(mongodbProperties.getApplicationName())
				.description(mongodbProperties.getDescription())
				.connectionsPerHost(mongodbProperties.getConnectionsPerHost())
				.minConnectionsPerHost(mongodbProperties.getMinConnectionsPerHost())
				.threadsAllowedToBlockForConnectionMultiplier(
						mongodbProperties.getThreadsAllowedToBlockForConnectionMultiplier())
				.cursorFinalizerEnabled(mongodbProperties.isCursorFinalizerEnabled())
				.maxWaitTime(mongodbProperties.getMaxWaitTime()).connectTimeout(mongodbProperties.getConnectTimeout())
				.socketTimeout(mongodbProperties.getSocketTimeout())
				.socketKeepAlive(mongodbProperties.isSocketKeepAlive())
				.threadsAllowedToBlockForConnectionMultiplier(100)
				.maxConnectionIdleTime(mongodbProperties.getMaxConnectionIdleTime())
				.maxConnectionLifeTime(mongodbProperties.getMaxConnectionLifeTime())
				.serverSelectionTimeout(mongodbProperties.getServerSelectionTimeout())
				.heartbeatSocketTimeout(mongodbProperties.getHeartbeatSocketTimeout())
				.heartbeatConnectTimeout(mongodbProperties.getHeartbeatConnectTimeout())
				.minHeartbeatFrequency(mongodbProperties.getMinHeartbeatFrequency())
				.heartbeatFrequency(mongodbProperties.getHeartbeatFrequency())
				.alwaysUseMBeans(mongodbProperties.isAlwaysUseMBeans()).build();
		this.properties.setUri(mongodbProperties.getUri());
		this.mongo = this.properties.createMongoClient(this.options, this.env);
		return this.mongo;
	}

	public class MongodbPropertiesConfig {

		@Bean(MongodbProperties.BEAN_NAME)
		public MongodbProperties mongodbProperties() {
			MongodbProperties mongodbProperties = new MongodbProperties();
			return mongodbProperties;
		}
	}
}
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.ComponentScan;

@ComponentScan
public class MongodbProperties {

	public static final String BEAN_NAME = "mongodbProperties";

	@Value("${mongo.uri}")
	private String uri;

	@Value("${mongo.applicationName}")
	private String applicationName;

	@Value("${mongo.description}")
	private String description;

	@Value("${mongo.connectionsPerHost}")
	private int connectionsPerHost;

	@Value("${mongo.minConnectionsPerHost}")
	private int minConnectionsPerHost;

	@Value("${mongo.threadsAllowedToBlockForConnectionMultiplier}")
	private int threadsAllowedToBlockForConnectionMultiplier;

	@Value("${mongo.cursorFinalizerEnabled}")
	private boolean cursorFinalizerEnabled;

	@Value("${mongo.maxWaitTime}")
	private int maxWaitTime;

	@Value("${mongo.connectTimeout}")
	private int connectTimeout;

	@Value("${mongo.socketTimeout}")
	private int socketTimeout;

	@Value("${mongo.socketKeepAlive}")
	private boolean socketKeepAlive;

	@Value("${mongo.maxConnectionIdleTime}")
	private int maxConnectionIdleTime;

	@Value("${mongo.maxConnectionLifeTime}")
	private int maxConnectionLifeTime;

	@Value("${mongo.serverSelectionTimeout}")
	private int serverSelectionTimeout;

	@Value("${mongo.heartbeatSocketTimeout}")
	private int heartbeatSocketTimeout;

	@Value("${mongo.heartbeatConnectTimeout}")
	private int heartbeatConnectTimeout;

	@Value("${mongo.minHeartbeatFrequency}")
	private int minHeartbeatFrequency;

	@Value("${mongo.heartbeatFrequency}")
	private int heartbeatFrequency;

	@Value("${mongo.alwaysUseMBeans}")
	private boolean alwaysUseMBeans;

	@Override
	public String toString() {
		return "MongodbProperties [uri=" + uri + ", applicationName=" + applicationName + ", description=" + description
				+ ", connectionsPerHost=" + connectionsPerHost + ", minConnectionsPerHost=" + minConnectionsPerHost
				+ ", threadsAllowedToBlockForConnectionMultiplier=" + threadsAllowedToBlockForConnectionMultiplier
				+ ", cursorFinalizerEnabled=" + cursorFinalizerEnabled + ", maxWaitTime=" + maxWaitTime
				+ ", connectTimeout=" + connectTimeout + ", socketTimeout=" + socketTimeout + ", socketKeepAlive="
				+ socketKeepAlive + ", maxConnectionIdleTime=" + maxConnectionIdleTime + ", maxConnectionLifeTime="
				+ maxConnectionLifeTime + ", serverSelectionTimeout=" + serverSelectionTimeout
				+ ", heartbeatSocketTimeout=" + heartbeatSocketTimeout + ", heartbeatConnectTimeout="
				+ heartbeatConnectTimeout + ", minHeartbeatFrequency=" + minHeartbeatFrequency + ", heartbeatFrequency="
				+ heartbeatFrequency + ", alwaysUseMBeans=" + alwaysUseMBeans + "]";
	}

	public String getApplicationName() {
		return applicationName;
	}

	public String getDescription() {
		return description;
	}

	public int getConnectionsPerHost() {
		return connectionsPerHost;
	}

	public int getMinConnectionsPerHost() {
		return minConnectionsPerHost;
	}

	public int getThreadsAllowedToBlockForConnectionMultiplier() {
		return threadsAllowedToBlockForConnectionMultiplier;
	}

	public boolean isCursorFinalizerEnabled() {
		return cursorFinalizerEnabled;
	}

	public int getMaxWaitTime() {
		return maxWaitTime;
	}

	public int getConnectTimeout() {
		return connectTimeout;
	}

	public int getSocketTimeout() {
		return socketTimeout;
	}

	public boolean isSocketKeepAlive() {
		return socketKeepAlive;
	}

	public int getMaxConnectionIdleTime() {
		return maxConnectionIdleTime;
	}

	public int getMaxConnectionLifeTime() {
		return maxConnectionLifeTime;
	}

	public int getServerSelectionTimeout() {
		return serverSelectionTimeout;
	}

	public int getHeartbeatSocketTimeout() {
		return heartbeatSocketTimeout;
	}

	public int getHeartbeatConnectTimeout() {
		return heartbeatConnectTimeout;
	}

	public int getMinHeartbeatFrequency() {
		return minHeartbeatFrequency;
	}

	public int getHeartbeatFrequency() {
		return heartbeatFrequency;
	}

	public boolean isAlwaysUseMBeans() {
		return alwaysUseMBeans;
	}

	public String getUri() {
		return uri;
	}
}

任务调度的方法，一般可以用quartz定时任务调取batch的任务 

import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.JobParametersInvalidException;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException;
import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException;
import org.springframework.batch.core.repository.JobRestartException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;


@RestController
@RequestMapping("/")
public class TestController {

	@Autowired
	private JobLauncher jobLauncher;
	
	@Autowired
        Job processJob1;
	// 任务开始调度
	@RequestMapping("/do")
	public String handle() throws JobExecutionAlreadyRunningException, JobRestartException, JobInstanceAlreadyCompleteException, JobParametersInvalidException {
		JobParameters jobParameter= new JobParametersBuilder().addLong("time", System.currentTimeMillis()).toJobParameters();
		jobLauncher.run(processJob1, jobParameter);
		 return "Batch job has been invoked";
	}

}

通用BatchConfig的配置

import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;

import java.util.concurrent.ThreadPoolExecutor;

@Configuration
public class BatchConfig {

    @Autowired
    protected JobBuilderFactory jobBuilderFactory;

    @Autowired
    protected StepBuilderFactory stepBuilderFactory;

    @Autowired
    protected RabbitTemplate amqpTemplate;

    @Autowired
    protected MongoTemplate mongoTemplate;

    //构造线程
    @Bean
    protected ThreadPoolTaskExecutor taskExecutor(){
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        executor.setCorePoolSize(10);
        executor.setMaxPoolSize(15);
        executor.setKeepAliveSeconds(300);
        executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardOldestPolicy());
        executor.setQueueCapacity(10000);
        executor.setThreadGroupName("spring_batch");
        return executor;
    }

}

再创建一个个性化的config类直接继承BatchConfig类

import com.xx.xx.config.BatchConfig;
import com.xx.xx.batch.dao.UserDao;
import com.xx.xx.batch.entity.User;
import com.xx.xx.batch.listener.UserJobCompletionListener;
import com.xx.xx.batch.processor.UserProcessor;
import com.xx.xx.batch.read.RabbitRead;
import com.xx.xx.batch.writer.MongoWriter;
import com.xx.xx.batch.writer.MysqlWriter;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecutionListener;
import org.springframework.batch.core.Step;
import org.springframework.batch.core.launch.support.RunIdIncrementer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import javax.annotation.Resource;


@Configuration
public class UserBatchConfig extends BatchConfig {

    @Resource
    private UserDao userDao;

    @Bean
    public Step userStep(){
        //此处writer如果要使用存储多个地方 可以直接参考getCompositeItemWriter方法 把getCompositeItemWriter 放入writer中就ok了
        return stepBuilderFactory.get("userStep").<String,String>chunk(10000)
                .reader(new RabbitRead<User>(amqpTemplate,"user_batch")).processor(new UserProcessor()).
                        writer(new MongoWriter(mongoTemplate,"user_batch")).taskExecutor(taskExecutor())
                .build();
    }
    @Bean
    public Job processJob1(){
        return jobBuilderFactory.get("processJob1").
                incrementer(new RunIdIncrementer()).listener(listener()).
                flow(userStep()).end().build();
    }
    public CompositeItemWriter<CrawlCommentDto> getCompositeItemWriter(){
        CompositeItemWriter<User> writers= new CompositeItemWriter();
        writers.setDelegates(Arrays.asList(new MysqlWriter<User>(userDao),new MongoWriter(mongoTemplate, "table_name")));
        return writers;
    }
    // 监听事件
    @Bean
    public JobExecutionListener listener() {
        return new UserJobCompletionListener();
    }
}

编写通用的读取消息队列的reader

import org.springframework.amqp.rabbit.core.RabbitTemplate;
import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter;
import org.springframework.amqp.support.converter.MessageConverter;
import org.springframework.batch.item.ItemReader;
import org.springframework.context.annotation.Bean;

/**
 * Description: 从rabbitmq中读取消息
 */
public class RabbitRead<T> implements ItemReader {

	// amqpTemplate 实例
	private RabbitTemplate amqpTemplate;

	// 消息队列名称
	private String mqName;

	private final Object lock = new Object();

	public RabbitRead(RabbitTemplate amqpTemplate, String mqName){
		this.amqpTemplate = amqpTemplate;
		this.mqName = mqName;
	}

	@Override
	public T read() {
		T receive;
                //加同步锁，以防万一
		synchronized (lock) {
			amqpTemplate.setMessageConverter(jsonMessageConverter());
			receive = (T) amqpTemplate.receiveAndConvert(mqName);
		}
		return receive;
	}
	@Bean
	public MessageConverter jsonMessageConverter() {
		return new Jackson2JsonMessageConverter();
	}

}

编写业务逻辑处理的类UserProcessor 

import org.springframework.batch.item.ItemProcessor;

public class UserProcessor implements ItemProcessor<User, User> {

	@Override
	public User process(User data) {
        System.out.print("开始处理用户信息.");
		return data;
	}

}

编写监听类

import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.listener.JobExecutionListenerSupport;

public class UserJobCompletionListener extends JobExecutionListenerSupport {
    @Override
    public void beforeJob(JobExecution jobExecution) {
        jobExecution.getJobId();
        System.out.println("job 开始啦");
        super.beforeJob(jobExecution);
    }

    @Override
    public void afterJob(JobExecution jobExecution) {
        super.afterJob(jobExecution);
    }
}

 springbatch可以有多个监听器，一共有6个，不同监听器有不同的用法，请参考实际添加监听器

编写通用的MongoWriter

import org.springframework.batch.item.data.MongoItemWriter;
import org.springframework.data.mongodb.core.MongoTemplate;

import java.util.List;

/**
 * Description: mongoWriter通用
 */
public class MongoWriter extends MongoItemWriter {

	public MongoTemplate mongoTemplate;

	// mongoDb表名
	private String mongoCollection;

	public MongoWriter(MongoTemplate mongoTemplate, String mongoCollection){
		this.mongoTemplate = mongoTemplate;
		this.mongoCollection = mongoCollection;
	}

	@Override
	public void write(List list) throws Exception{
		super.setCollection(mongoCollection);
		super.setTemplate(mongoTemplate);
		super.write(list);
	}
}

编写通用的MysqlWriter

import com.xx.xx.batch.dao.MysqlCommonDao;
import org.springframework.batch.item.ItemWriter;

import java.util.List;

/**
 * Description: mysqlWriter通用
 */
public class MysqlWriter<T> implements ItemWriter<T> {

	private MysqlCommonDao mysqlCommonDao;

	public MysqlWriter(MysqlCommonDao mysqlCommonDao){
		this.mysqlCommonDao = mysqlCommonDao;
	}

	@Override
	public void write(List<? extends T> items) {
		items.forEach(action -> {
			mysqlCommonDao.add(action);
			System.out.println(action);
		});
	}
}

通用的dao在使用时直接继承

import java.util.List;

/**
 * Description: mysql通用dao
 */
public interface MysqlCommonDao<T> {

    Integer add(T t);

    List select();

    Integer update();

    Integer delete();

}

@Mapper
public interface UserDao<T> extends MysqlCommonDao<T>{

    @Override
    Integer add(T user);
}

 实体类：

public class User {
    private int id;

    private int age;

    private String name;

    public int getId() {
        return id;
    }

    public void setId(int id) {
        this.id = id;
    }

    public int getAge() {
        return age;
    }

    public void setAge(int age) {
        this.age = age;
    }

    public String getName() {
        return name;
    }

    public void setName(String name) {
        this.name = name;
    }
}

mapper文件：

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.xx.xx.batch.dao.UserDao">
    <insert id="add" parameterType="com.xx.xx.batch.entity.User">
        insert into user_batch_test values(#{id},#{age},#{name})
    </insert>
</mapper>

最后在springboot启动类上添加注解

@EnableBatchProcessing
@SpringBootApplication
@EnableAsync
@EnableDiscoveryClient
@MapperScan(basePackages = "com.xx.xx.batch.dao")
@ComponentScan(basePackages = { "com.xx.xx.xx"})

至此大功告成。

warmer_winter
关注
  • 3
    点赞
  • 踩
  • 5
    收藏
    觉得还不错? 一键收藏
  • 8
    评论
专栏目录
评论 8
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值