mybatis多线程,oracle,百万数据从一个表导入另一个表

要求:
从几个表读取数据,存入另几个表,这几个表一一对应,但是字段名不相同,我的情况是被读取表包含需要的所有数据,我按天取数据,每天400多万条。
思路:
先是读取数据,分页读,一次20000条,然后批量存(一次1000),然后每次导表都是一个线程
实现:
serviceImpl:

//存数据
public void insert1(List<xxx> xxxList,String j){
		List<Map<String,Object>> list = new ArrayList<>();
		for(xxx x: xxxList){
			x.setId(UUID.randomUUID().toString().replaceAll("-",""));
			Map<String,Object> h = new HashMap<>();
			h.put("id",x.getId());
			...
			list.add(h);
		}
		//20000个分成20组,一次提交1000个
		List<List<Map<String,Object>>> l = Lists.partition(list,20);
		if (!l.isEmpty()) {
			for (List<Map<String, Object>> maps : l) {
				if (maps != null && !maps.isEmpty()) {
					xxxMapper.insertx(maps,j);
				}
			}
		}
	}

	/**
	 * 读数据,分页,一次20000个
	 * @param j
	 */
	public void findList1(String j){
		int pageSize1 = 20000;
		Map<String, Object> a = new HashMap<>();
		a.put("sDay","2021-08-"+j+" 00:00:00");
		a.put("eDay","2021-08-"+j+" 23:59:59");
		int size = xxxMapper.findCount(a);
		int cs = size/pageSize1+1;
		for (int i=1; i<=cs;i++) {
			PageHelper.startPage(i, pageSize1);
			List<xxx> xxxList = xxxMapper.selectx(a);
			SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss");
			String sj = sdf.format(new Date());
			if (xxxList!=null && !xxxList.isEmpty()) {
				insert1(xxxList,j);
				System.out.println(j+"!!!!!!!!!!共"+cs+"次,----第"+i+"次,时间"+sj);
			}
		}
	}
	
	service:
	void findList1(String j);

controller:

	public void ttt(){
		ThreadPoolService.run(new TestService(Service,"05"));
		ThreadPoolService.run(new TestService(Service,"06"));
		ThreadPoolService.run(new TestService(Service,"07"));
		ThreadPoolService.run(new TestService(Service,"08"));
		ThreadPoolService.run(new TestService(Service,"09"));
		ThreadPoolService.run(new TestService(Service,"10"));
		ThreadPoolService.run(new TestService(Service,"11"));
		ThreadPoolService.run(new TestService(Service,"12"));
	}

线程池:

package common.util;

import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

public class ThreadPoolService {
	
	private static final ThreadPoolExecutor threadpool =  new ThreadPoolExecutor(5, 5,
            30L, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
	
	
    public static void run(Runnable command) {
		threadpool.execute(command);
	} 
    public static Future<?> submit(Runnable command) {
		return threadpool.submit(command);
	}
	public static int getActiveCount() {
		return threadpool.getActiveCount();
	}
	public static int getQueueSize() {
		return threadpool.getQueue().size();
	} 
}

TestService:

package zd.np.service;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.UnsupportedEncodingException;


public class TestService implements Runnable{

	private Logger logger = LogManager.getLogger(TestService.class);
	private Service service;
	private String j;

	private TestService() { }

	public TestService(Service service,String j) {
		this.service=service;
		this.j = j;
	}


	@Override
	public synchronized void run() {
		try {
			System.out.println(Thread.currentThread().getName()+"正在执行....");
			//每个线程都是一次读和存数据
			service.findList1(j);
			System.out.println(Thread.currentThread().getName()+"结束执行....");
		} catch (Exception e) {
			logger.error(e.getMessage(),e);
		}finally {
			clear();
		}
	}


	private void clear() {
		this.logger=null;
		this.Service=null;
		this.j=null;
	}
	public static void main(String[] args) throws UnsupportedEncodingException {
	}
}

mapper:

void insertx(List<Map<String,Object>> list,@Param("day") String day);
	List<xxx> selectx(Map<String,Object> x);

mapper.xml:

<insert id="insertx" parameterType="java.util.List">
		INSERT INTO 表名${day}(id,...)
		<foreach collection="list" item="item" separator="UNION ALL">
			select
			#{item.id,jdbcType=VARCHAR},...
			from DUAL
		</foreach>
</insert>
<select id="selectx" resultMap="ResultMaphhh" parameterType="hashmap">
		select xx字段名
		from xxx表名
		where sjgxsj &gt;= to_date(#{sDay},'yyyy-mm-dd hh24:mi:ss')
		and  sjgxsj &lt;= to_date(#{eDay},'yyyy-mm-dd hh24:mi:ss')
</select>
<select id="findCount" resultType="java.lang.Integer" parameterType="hashmap">
	select count(1)
	from xxx表名
	where sj &gt;= to_date(#{sDay},'yyyy-mm-dd hh24:mi:ss')
	  and  sj &lt;= to_date(#{eDay},'yyyy-mm-dd hh24:mi:ss')
</select>
<resultMap id="ResultMaphhh" type="xx.xx.model.xxx" >
	<result column="???" property="???" />
</resultMap>
<dependency>
	<groupId>com.google.guava</groupId>
	<artifactId>guava</artifactId>
	<version>21.0</version>
</dependency>

附带一个存储过程的方法,作为参考,与上面的东西无关

create or replace procedure copyTo is
  -- 定义游标,并将sql结果集赋值到游标中
    CURSOR report is select 字段 from 读取表名;
    CUR report%ROWTYPE; --定义游标变量,该变量的类型为基于游标C_EMP的记录
    xxxid varchar(40);
begin
    -- 打开游标
    open report;
       LOOP
        FETCH report INTO CUR;
        EXIT WHEN report%NOTFOUND;
        --给xxxid赋值uuid
        select sys_guid() into xxxid from dual;
        -- 执行业务逻辑
        INSERT INTO 存入表名(id,...)
        VALUES(xxxid,...);
      END LOOP;
    -- 关闭游标
    close report;
end copyTo;
评论 1
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值