需要先引入解析工具
<!--cvs 解析工具-->
<dependency>
<groupId>com.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>4.3.2</version>
</dependency>
抽取出工具类
package io.renren.utils;
import com.opencsv.bean.CsvToBean;
import com.opencsv.bean.CsvToBeanBuilder;
import com.opencsv.bean.HeaderColumnNameMappingStrategy;
import org.springframework.web.multipart.MultipartFile;
import java.io.InputStreamReader;
import java.util.List;
public class cvsUtils {
public static <T> List<T> getCsvData(MultipartFile file, Class<T> clazz) {
InputStreamReader in = null;
try {
in = new InputStreamReader(file.getInputStream(), "gbk");
} catch (Exception e) {
System.out.println("读取文件失败");
}
HeaderColumnNameMappingStrategy<T> strategy = new HeaderColumnNameMappingStrategy<>();
strategy.setType(clazz);
CsvToBean<T> csvToBean = new CsvToBeanBuilder<T>(in)
.withSeparator(',')
.withQuoteChar('\'')
.withMappingStrategy(strategy).build();
return csvToBean.parse();
}
}
案例demo
package io.renren.entity.csv;
import com.opencsv.bean.CsvBindByName;
public class User {
@CsvBindByName(column = "id")
private Long id;
/**
* 字典代码
*/
@CsvBindByName(column = "code")
private String code;
/**
* 简写
*/
@CsvBindByName(column = "shortName")
private String shortName;
/**
* N
* 名称
*/
@CsvBindByName(column = "name") //是否可以为null 否
private String name;
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getShortName() {
return shortName;
}
public void setShortName(String shortName) {
this.shortName = shortName;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
}
下面是核心代码
使用线程池和java8Stream流数据分片去处理
配置好线程池 和 CompletableFuture 去做异步处理
线程池配置
package io.renren.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import java.util.concurrent.ThreadPoolExecutor;
@Configuration
@EnableAsync
public class ThreadPoolTaskExecutorConfig {
@Bean("taskModuleExecutor")
ThreadPoolTaskExecutor getCrawler1(){
ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor();
threadPoolTaskExecutor.setCorePoolSize(5);
threadPoolTaskExecutor.setMaxPoolSize(6);
threadPoolTaskExecutor.setQueueCapacity(200);
threadPoolTaskExecutor.setThreadNamePrefix("task-concurrent-work");
threadPoolTaskExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
threadPoolTaskExecutor.initialize();
return threadPoolTaskExecutor;
}
}```
//每一千条批量插入 (这里根据自己的需求去定义)
public static final long PAGE_SIZE =1000;
@PostMapping("/uploadVCSBeach")
public R uploadVCSBeachThere(MultipartFile file) throws IOException {
//cvs工具类解析数据
List<User> userList = cvsUtils.getCsvData(file, User.class);
if (CollectionUtils.isNotEmpty(userList)){
//总页数
long count = userList.size()/PAGE_SIZE;
//遍历每一页
for (Integer i = 1; i < count+1; i++) {
int n =i;
// 这里是做分片
List<User> subUserList = userList.stream()
.skip((n - 1) * PAGE_SIZE) //方法用于跳过前n条数据
.limit(PAGE_SIZE) //方法用于返回前n条数据,
.collect(Collectors.toList());
try {
CompletableFuture<Void> voidCompletableFuture = CompletableFuture.runAsync(() -> {
// 批量插入
userDao.insertBatch(subUserList);
}, threadPoolExecutor);
// CompletableFuture.allOf(voidCompletableFuture).get();
} catch (Exception e) {
e.printStackTrace();
}
}
}
return R.ok();
}