多线程批量操作
OperateService 多线程操作
public interface OperateService {
/**
* 多线程导入
* @param list
* @param consumer
* @param context
* @param <T>
*/
<T> void multiOperate(List<T> list,Consumer<List<T>> consumer, OperatorContext context);
/**
*
* @param list
* @param consumer
* @param context
* @param <T>
*/
<T> void multiOperate2(List<T> list,Consumer<List<T>> consumer, OperatorContext2 context);
}
DataSourceModelService 数据库操作
public interface DataSourceModelService extends IService<DataSourceModel> {
/**
* 子线程处理导入
* @param list
* @param context
*/
void dataImport(List<DataSourceModel> list,OperatorContext context);
/**
* 子线程处理导入2
* @param dataSourceModels
* @param context
*/
void dataImport2(List<DataSourceModel> dataSourceModels, OperatorContext2 context);
}
OperatorContextHolder 事务上下文管理器
/**
* @Date: 2022/10/11 14:44
* 事务上下文管理
*/
@Component
public class OperatorContextHolder {
@Autowired
private PlatformTransactionManager transactionManager;
/**
* 创建一个新匆事务上下文
*
* @return
*/
public OperatorContext newContext(int threads) {
return new OperatorContext(transactionManager, threads);
}
/**
* 手动 创建事务上下文
*
* @param threads
* @return
*/
public OperatorContext2 newContext2(int threads) {
return new OperatorContext2(transactionManager, threads);
}
}
每个子线程单独提交事务
OperatorContext
/**
* @Date: 2022/10/11 14:44
* 事务上下文
*/
public class OperatorContext {
/**
* 多线程处理过程中是否有异常
*/
private volatile boolean error;
private CountDownLatch main;
private CountDownLatch sub;
private PlatformTransactionManager transactionManager;
public OperatorContext(PlatformTransactionManager transactionManager, int threads) {
this.transactionManager = transactionManager;
main = new CountDownLatch(1);
sub = new CountDownLatch(threads);
}
public void setError() {
this.error = true;
}
public boolean isError() {
return error;
}
/**
* 手动 开启一个新事务
*
* @return
*/
public TransactionStatus newTransaction() {
TransactionDefinition definition = new DefaultTransactionDefinition();
return transactionManager.getTransaction(definition);
}
public CountDownLatch getMain() {
return main;
}
public CountDownLatch getSub() {
return sub;
}
}
DataSourceModelServiceImpl
@Service
@Slf4j
public class DataSourceModelServiceImpl implements DataSourceModelService {
@Autowired
private PlatformTransactionManager transactionManager;
/**
* 1.每个子线程都会开启自己的事务
* 2.如果子线程事务执行异常，让每个子线程事务回滚
*
* @param list
* @param context
*/
@Override
public void dataImport(List<DataSourceModel> list, OperatorContext context) {
//开启新事务
TransactionStatus transaction = context.newTransaction();
CountDownLatch main = context.getMain();
CountDownLatch sub = context.getSub();
log.info("{} is start processing transaction {} ...", Thread.currentThread().getName(), transaction.hashCode());
try {
list.stream().forEach(model -> {
if (model.getDatabaseName().equals("abc")) {
String format = MessageFormat.format("{0} transaction {1} 异常啦，事务需要回滚", Thread.currentThread().getName(), transaction.hashCode());
throw new RuntimeException(format);
}
baseMapper.insert(model);
});
} catch (Exception e) {
log.info("{} transaction {} catch exception ...", Thread.currentThread().getName(), transaction.hashCode());
context.setError();
} finally {
log.info("{} finally transaction {} ...", Thread.currentThread().getName(), transaction.hashCode());
sub.countDown();
}
try {
//阻塞 所有 子线程运行结束
//等等执行 main.countDown()操作
main.await();
} catch (InterruptedException e) {
context.setError();
}
log.info("{} continue transaction {} ...", Thread.currentThread().getName(), transaction.hashCode());
if (!context.isError()) {
//回滚每个子线程事务
log.info("{} commit transaction {}...", Thread.currentThread().getName(), transaction.hashCode());
transactionManager.commit(transaction);
} else {
log.info("{} rollback transaction {}...", Thread.currentThread().getName(), transaction.hashCode());
transactionManager.rollback(transaction);
}
}
}
OperateServiceImpl
@Slf4j
@Service
public class OperateServiceImpl implements OperateService {
/**
* 多线程导入
* @param list
* @param consumer
* @param context
* @param <T>
*/
@Override
public <T> void multiOperate(List<T> list, Consumer<List<T>> consumer, OperatorContext context) {
//多线程执行
consumer.accept(list);
CountDownLatch main = context.getMain();
CountDownLatch sub = context.getSub();
try {
sub.await(30, TimeUnit.SECONDS);
} catch (InterruptedException e) {
context.setError();
}
//等待所有线下执行，最后触发每个线程中提交事务的动作
main.countDown();
log.info("{} 主线程执行完成...", Thread.currentThread().getName());
}
}
api层
@Slf4j
@RestController
public class DataSourceController {
@Autowired
private DataSourceModelService dataSourceModelService;
@Autowired
private OperatorContextHolder contextHolder;
@Autowired
private ThreadPoolTaskExecutor taskExecutor;
@Autowired
private OperateService operateService;
@PostMapping(value = "/v1.0/ds/import", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public Boolean importData(MultipartFile file) throws IOException {
SyncReadListener syncReadListener = new SyncReadListener();
EasyExcel.read(file.getInputStream(), DataSourceDto.class, syncReadListener).sheet().doRead();
List<DataSourceModel> list = syncReadListener.getList().stream().map(m -> {
DataSourceModel model = new DataSourceModel();
BeanUtils.copyProperties(m, model);
return model;
}).collect(Collectors.toList());
int page = 3;
//计算线程
List<List<DataSourceModel>> partition = Lists.partition(list, page);
//创建多线程事务上下文
OperatorContext context = contextHolder.newContext(partition.size());
operateService.multiOperate(partition,val -> {
for (List<DataSourceModel> dataSourceModels : val) {
taskExecutor.execute(() -> dataSourceModelService.dataImport(dataSourceModels,context));
}
}, context);
return true;
}
}
访问 http://localhost:8082/v1.0/ds/import
测试回滚
task-14 is start processing transaction 2067321468 ...
task-13 is start processing transaction 2131372607 ...
task-14 transaction 2067321468 catch exception ...
task-14 finally transaction 2067321468 ...
task-13 finally transaction 2131372607 ...
http-nio-8082-exec-6 主线程执行完成...
task-13 continue transaction 2131372607 ...
task-14 continue transaction 2067321468 ...
task-13 rollback transaction 2131372607...
task-14 rollback transaction 2067321468...
测试提交
task-11 is start processing transaction 1088629827 ...
task-12 is start processing transaction 529406267 ...
task-11 finally transaction 1088629827 ...
task-12 finally transaction 529406267 ...
http-nio-8082-exec-3 主线程执行完成...
task-11 continue transaction 1088629827 ...
task-12 continue transaction 529406267 ...
task-11 commit transaction 1088629827...
task-12 commit transaction 529406267...
主线程批量提交事务
OperatorContext2 事务上下文
/**
* @Date: 2022/10/11 14:44
* 事务上下文
*/
@Slf4j
public class OperatorContext2 {
/**
* 多线程处理过程中是否有异常
*/
private volatile boolean error;
private CountDownLatch main;
private PlatformTransactionManager transactionManager;
private List<TransactionStatus> list;
public OperatorContext2(PlatformTransactionManager transactionManager, int threads) {
this.transactionManager = transactionManager;
main = new CountDownLatch(threads);
list = new ArrayList<>();
}
public void setError() {
this.error = true;
}
public boolean isError() {
return error;
}
/**
* 手动 开启一个新事务
*
* @return
*/
public TransactionStatus startTransaction() {
DefaultTransactionDefinition definition = new DefaultTransactionDefinition();
definition.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);
TransactionStatus transaction = transactionManager.getTransaction(definition);
list.add(transaction);
return transaction;
}
public CountDownLatch getMain() {
return main;
}
/**
* 事务批量提交
*/
public void commitTransactions() {
if (CollectionUtils.isEmpty(list)) {
return;
}
for (TransactionStatus transaction : list) {
log.info("{} commit transaction {}...", Thread.currentThread().getName(), transaction.hashCode());
transactionManager.commit(transaction);
}
}
/**
* 事务 批量回滚
*/
public void rollbackTransactions() {
if (CollectionUtils.isEmpty(list)) {
return;
}
for (TransactionStatus transaction : list) {
log.info("{} rollback transaction {}...", Thread.currentThread().getName(), transaction.hashCode());
transactionManager.rollback(transaction);
}
}
}
OperateServiceImpl
@Slf4j
@Service
public class OperateServiceImpl implements OperateService {
/**
* 主线程方法需要加 事务
* @param list
* @param consumer
* @param context
* @param <T>
*/
@Override
@Transactional
public <T> void multiOperate2(List<T> list, Consumer<List<T>> consumer, OperatorContext2 context) {
consumer.accept(list);
CountDownLatch main = context.getMain();
try {
log.info("{} await ...", Thread.currentThread().getName());
boolean await = main.await(30, TimeUnit.SECONDS);
if (!await) {
context.setError();
}
} catch (InterruptedException e) {
context.setError();
}
log.info("{} prepare rollback or commit transactions ...", Thread.currentThread().getName());
if (!context.isError()) {
//回滚每个子线程事务
log.info("{} commit transactions ...", Thread.currentThread().getName());
context.commitTransactions();
} else {
log.info("{} rollback transactions ...", Thread.currentThread().getName());
context.rollbackTransactions();
}
log.info("{} 主线程执行完成...", Thread.currentThread().getName());
}
}
DataSourceModelServiceImpl
@Service
@Slf4j
public class DataSourceModelServiceImpl implements DataSourceModelService {
/**
* 2.加事务,不然报错！！
* @param list
* @param context
*/
@Override
@Transactional
public void dataImport2(List<DataSourceModel> list, OperatorContext2 context) {
//开启新事务
TransactionStatus transaction = context.startTransaction();
CountDownLatch main = context.getMain();
log.info("{} is start processing transaction {} ...", Thread.currentThread().getName(), transaction.hashCode());
try {
list.stream().forEach(model -> {
if (model.getDatabaseName().equals("abc")) {
String format = MessageFormat.format("{0} transaction {1} 异常啦，事务需要回滚", Thread.currentThread().getName(), transaction.hashCode());
throw new RuntimeException(format);
}
baseMapper.insert(model);
});
} catch (Exception e) {
log.info("{} transaction {} catch exception ...", Thread.currentThread().getName(), transaction.hashCode());
context.setError();
} finally {
log.info("{} finally transaction {} ...", Thread.currentThread().getName(), transaction.hashCode());
main.countDown();
}
}
}
api 层
@PostMapping(value = "/v1.0/ds/import2", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public Boolean importData2(MultipartFile file) throws IOException {
SyncReadListener syncReadListener = new SyncReadListener();
EasyExcel.read(file.getInputStream(), DataSourceDto.class, syncReadListener).sheet().doRead();
List<DataSourceModel> list = syncReadListener.getList().stream().map(m -> {
DataSourceModel model = new DataSourceModel();
BeanUtils.copyProperties(m, model);
return model;
}).collect(Collectors.toList());
int page = 3;
//计算线程
List<List<DataSourceModel>> partition = Lists.partition(list, page);
//创建多线程事务上下文
OperatorContext2 context = contextHolder.newContext2(partition.size());
operateService.multiOperate2(partition,val -> {
for (List<DataSourceModel> dataSourceModels : val) {
taskExecutor.execute(() -> dataSourceModelService.dataImport2(dataSourceModels,context));
}
}, context);
return true;
}
访问 http://localhost:8082/v1.0/ds/import2
测试回滚
http-nio-8082-exec-1 await ... # 主线程等待
task-2 is start processing transaction 2069003473 ...
task-1 is start processing transaction 1389075684 ...
task-2 transaction 2069003473 catch exception ...
task-2 finally transaction 2069003473 ...
task-1 finally transaction 1389075684 ...
http-nio-8082-exec-1 prepare rollback or commit transactions ...
http-nio-8082-exec-1 rollback transactions ...
http-nio-8082-exec-1 rollback transaction 2069003473... # 主线程批量回滚
http-nio-8082-exec-1 rollback transaction 1389075684...
http-nio-8082-exec-1 主线程执行完成...
测试提交
http-nio-8082-exec-5 await ...
task-3 is start processing transaction 2085237889 ...
task-4 is start processing transaction 1124059160 ...
task-4 finally transaction 1124059160 ...
task-3 finally transaction 2085237889 ...
http-nio-8082-exec-5 prepare rollback or commit transactions ...
http-nio-8082-exec-5 commit transactions ...
http-nio-8082-exec-5 commit transaction 2085237889...
http-nio-8082-exec-5 commit transaction 1124059160...
http-nio-8082-exec-5 主线程执行完成...
good luck!
扫一扫
422
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
