分布式任务-数据批量新增数据库

分布式-数据批量新增数据库

0.pojo

package com.mall.order.pojo;
import org.apache.commons.lang3.builder.ToStringBuilder;
import javax.persistence.*;
import java.io.Serializable;
import java.lang.Long;
import java.util.Date;
import java.lang.String;
import java.lang.Integer;

/****
 * 订单明细信息表 order_item_details
 *****/
@Table(name="order_item_details")
public class OrderItemDetails implements Serializable{

	@Id
    @GeneratedValue(strategy = GenerationType.IDENTITY) //primary keys for the entity using database identity column
    @Column(name = "id")
	private Long id;//唯一标识

    @Column(name = "order_no")
	private String orderNo;//订单编号

    @Column(name = "order_time")
	private Date orderTime;//订单提交时间

    @Column(name = "finish_time")
	private Date finishTime;//订单完成(确认收货)时间

    @Column(name = "total_price")
	private String totalPrice;//订单商品总金额

    @Column(name = "sku_id")
	private Long skuId;//采购平台skuId

    @Column(name = "item_name")
	private String itemName;//商品名称

    @Column(name = "sku_num")
	private Integer skuNum;//sku最终数量

    @Column(name = "brand_name")
	private String brandName;//品牌名称

    @Column(name = "policy_attributes")
	private String policyAttributes;//政策属性：{"is_jieneng":1,"is_huanbao":0}

    @Column(name = "is_jieneng")
	private String isJieneng;//是否节能

    @Column(name = "is_huanbao")
	private String isHuanbao;//是否环保

    @Column(name = "is_prison_enter")
	private String isPrisonEnter;//供应商是否监狱企业

    @Column(name = "order_type")
	private String orderType;//采购方式

    @Column(name = "purchasing_zone")
	private String purchasingZone;//采购专区

    @Column(name = "legal_entity_organization_name")
	private String legalEntityOrganizationName;//用户所属法人单位组织机构名称/采购单位

	@Column(name = "buyer_id")
	private Long buyerId;//采购人ID

	@Column(name = "buyer_name")
	private String buyerName;//采购人名称

	@Column(name = "supplier_id")
	private Long supplierId;//供应商ID

	@Column(name = "supplier_name")
	private String supplierName;//供应商名称

    @Column(name = "service_attitude_rating")
	private Integer serviceAttitudeRating;//服务态度 1~5星(采购人评价商家货物类商品 评星不能为空)

    @Column(name = "package_quality_rating")
	private Integer packageQualityRating;//包装质量 1~5星(采购人评价商家货物类商品 评星不能为空)

    @Column(name = "arrival_speed_rating")
	private Integer arrivalSpeedRating;//到货速度 1~5星(采购人评价商家货物类商品 评星不能为空)

    @Column(name = "process_effective_rating")
	private Integer processEffectiveRating;//流程高效 1~5星(采购人评价商家服务类商品 评星不能为空)

    @Column(name = "satisfaction_level_rating")
	private Integer satisfactionLevelRating;//满意程度 1~5星(采购人评价商家服务类商品 评星不能为空)

    @Column(name = "service_overall_rating")
	private String serviceOverallRating;//供应商(代理商)综合评分

    @Column(name = "satisfaction_rating")
	private Integer satisfactionRating;//服务满意度1~5星(采购人评价供应商，评星不能为空)

    @Column(name = "signin_timely_rating")
	private Integer signinTimelyRating;//签收及时性 1~5星(供应商评价采购人 评星不能为空)

    @Column(name = "paying_timely_rating")
	private Integer payingTimelyRating;//付款及时性 1~5星(供应商评价采购人 评星不能为空)

    @Column(name = "communication_rating")
	private Integer communicationRating;//沟通效率   1~5星(供应商评价采购人 评星不能为空)

    @Column(name = "overall_rating")
	private String overallRating;//综合评分(由"签收/付款/沟通"三项计算获得) 1~5星

    @Column(name = "remark1")
	private String remark1;//备注1

    @Column(name = "remark2")
	private String remark2;//备注2

    @Column(name = "remark3")
	private String remark3;//备注3

    @Column(name = "remark4")
	private String remark4;//备注4

    @Column(name = "remark5")
	private String remark5;//备注5

	// set ... get ...
	
}

1.OrderItemDetailsMapper.xml

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.mall.order.mapper.OrderItemDetailsMapper">

    <!--批量导入  封装sql查询已有数据 -准备封装入库order_item_details-->
    <insert id="importOrderItemAssembleDataBatch" parameterType="com.mall.order.pojo.OrderItemDetails"
            useGeneratedKeys="true" keyProperty="id">
        INSERT INTO order_item_details
            (
                order_no,
                order_time,
                finish_time,
                total_price,
                sku_id,
                item_name,
                sku_num,
                brand_name,
                policy_attributes,
                is_jieneng,
                is_huanbao,
                is_prison_enter,
                order_type,
                purchasing_zone,
                legal_entity_organization_name,
                buyer_id,
                buyer_name,
                supplier_id,
                supplier_name,
                service_attitude_rating,
                package_quality_rating,
                arrival_speed_rating,
                process_effective_rating,
                satisfaction_level_rating,
                service_overall_rating,
                satisfaction_rating,
                signin_timely_rating,
                paying_timely_rating,
                communication_rating,
                overall_rating,
                remark1,
                remark2,
                remark3,
                remark4,
                remark5
            )
        VALUES
            <foreach collection="list" item="item" separator="," index="index">
            (
                #{item.orderNo},
                #{item.orderTime},
                #{item.finishTime},
                #{item.totalPrice},
                #{item.skuId},
                #{item.itemName},
                #{item.skuNum},
                #{item.brandName},
                #{item.policyAttributes},
                #{item.isJieneng},
                #{item.isHuanbao},
                #{item.isPrisonEnter},
                #{item.orderType},
                #{item.purchasingZone},
                #{item.legalEntityOrganizationName},
                #{item.buyerId},
                #{item.buyerName},
                #{item.supplierId},
                #{item.supplierName},
                #{item.serviceAttitudeRating},
                #{item.packageQualityRating},
                #{item.arrivalSpeedRating},
                #{item.processEffectiveRating},
                #{item.satisfactionLevelRating},
                #{item.serviceOverallRating},
                #{item.satisfactionRating},
                #{item.signinTimelyRating},
                #{item.payingTimelyRating},
                #{item.communicationRating},
                #{item.overallRating},
                #{item.remark1},
                #{item.remark2},
                #{item.remark3},
                #{item.remark4},
                #{item.remark5}
            )
            </foreach>
    </insert>
</mapper>

2.OrderItemDetailsMapper.java

package com.mall.order.mapper;

import com.mall.order.pojo.OrderItemDetails;
import org.apache.ibatis.annotations.Param;
import tk.mybatis.mapper.common.Mapper;

import java.util.List;
import java.util.Map;

public interface OrderItemDetailsMapper extends Mapper<OrderItemDetails> {
  
    /**
     * 批量导入  封装sql查询已有数据 -准备封装入库order_item_details
     *
     * @param list
     * @return
     */
    int importOrderItemAssembleDataBatch(@Param("list") List<OrderItemDetails> list);
}

3.OrderItemDetailsServiceImpl.java

   /**
	* 批量导入已有数据 入库order_item_details
	*
	* @param parent_no
	* @param order_status
	*/
	void initOrderItemDetailsByfindAssembleData();

	@Autowired
    private OrderItemDetailsMapper orderItemDetailsMapper;
    
	/**
     * 批量导入已有数据 入库order_item_details
     * @param parent_no         0:第一级主单(拆单和未拆单都为0)，(!=0):当前订单的父单号（和order_split_type 配合查询订单）
     * @param order_status      订单状态：10待确认 20待发货 25部分发货 30已发货 40待收货 50已完成 65取消中 70已取消
     */
    @Override
    public void initOrderItemDetailsByfindAssembleData() {
       List<OrderItemDetails> details = null; //暂时置为null
       int batchNum = 300; //默认：每次批量操作的数据的个数
       DistributedMultithreading task = new DistributedMultithreading(orderItemDetailsMapper, details, batchNum);
       try {
           task.invoke();
           logger.info("批量添加订单明细数据完成！");
       } catch (Exception e) {
           e.printStackTrace();
       }
       return;
    }

4.DistributedMultithreading.java

package com.mall.order.service.handler;

import cn.hutool.core.collection.CollUtil;
import com.mall.order.mapper.OrderItemDetailsMapper;
import com.mall.order.pojo.OrderItemDetails;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;

/**
 * 分布式--批量操作数据
 *
 */
public class DistributedMultithreading {

    private final static Logger logger = LoggerFactory.getLogger(DistributedMultithreading.class);

    private OrderItemDetailsMapper orderItemDetailsMapper;
    private List<OrderItemDetails> list; //数据
    private int batchNum;  //每次批量操作的数据的个数

    public DistributedMultithreading(OrderItemDetailsMapper orderItemDetailsMapper, List<OrderItemDetails> list, int batchNum) {
            this.orderItemDetailsMapper = orderItemDetailsMapper;
            this.list = list;
            this.batchNum = batchNum;
    }

    /**
     * 开始执行
     *
     * @throws Exception
     */
    public void invoke() throws Exception {
        int totalNum = list.size();
        if (totalNum <= batchNum) { //只需要操作数据库一次
            handler(list);
            return;
        }
        //分布式--批量操作数据
        batchDeal(list, batchNum);
    }

    /**
     * 操作数据
     *
     * @param handlerList
     */
    protected void handler(List<OrderItemDetails> handlerList) {
        int result = orderItemDetailsMapper.importOrderItemAssembleDataBatch(handlerList);
        if (result > 0) {
            logger.info("共导入：{}条数据！", result);
            List<Long> new_ids = handlerList.stream().map(OrderItemDetails::getId).collect(Collectors.toList());
            logger.info("新导入数据的主键id为：{}", new_ids);
        }
    }

    /**
     * 分布式--批量操作数据
     *
     * @param data
     * @param batchNum
     * @throws Exception
     */
    protected void batchDeal(List<OrderItemDetails> data, int batchNum) throws Exception {
        int totalNum = data.size();
        int pageNum = totalNum % batchNum == 0 ? totalNum / batchNum : totalNum / batchNum + 1;
        ExecutorService executor = Executors.newFixedThreadPool(pageNum);
        try {
            CountDownLatch countDownLatch = new CountDownLatch(pageNum);
            List<OrderItemDetails> subData = null;
            int fromIndex, toIndex;
            int j = 0;
            while (j < pageNum) {
                fromIndex = j * batchNum;
                toIndex = Math.min(totalNum, fromIndex + batchNum);
                subData = data.subList(fromIndex, toIndex);
                ImportTask task = new ImportTask(subData, countDownLatch);
                executor.execute(task);
                j++;
            }
            // 主线程必须在启动其它线程后立即调用CountDownLatch.await()方法，
            // 这样主线程的操作就会在这个方法上阻塞，直到其它线程完成各自的任务。
            // 计数器的值等于0时，主线程就能通过await()方法恢复执行自己的任务。
            countDownLatch.await();
        } finally {
            // 关闭线程池，释放资源
            executor.shutdown();
        }
    }

    /**
     * 具体操作 实现类
     *
     */
    public class ImportTask implements Runnable {
        private List<OrderItemDetails> handlerList;
        private CountDownLatch countDownLatch;

        public ImportTask(List<OrderItemDetails> handlerList, CountDownLatch countDownLatch) {
            this.handlerList = handlerList;
            this.countDownLatch = countDownLatch;
        }

        @Override
        public void run() {
            if (CollUtil.isNotEmpty(handlerList)) {
                //批量insert或者update
                handler(handlerList);
            }
            countDownLatch.countDown(); //发出线程任务完成的信号
        }
    }
}

4.启动类

package com.mall;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.retry.annotation.EnableRetry;
import tk.mybatis.spring.annotation.MapperScan;

@SpringBootApplication
@MapperScan(value = {"com.mall.order.mapper"})
public class OrderItemDetailsApplication {
    public static void main(String[] args) {
        SpringApplication.run(OrderItemDetailsApplication.class,args);
    }
}