Disruptor简单应用

一、Disruptor是啥？

        一个无锁的高并发框架，环形的数据结构——直接覆盖（不用清除）旧的数据、大小必须是2的幂次方，降低GC频率
实现了基于事件的生产者消费者模式（观察者模式），实现同一产品被多个不同业务的消费者同时消费的业务。

二、简单的业务场景：

        多部门考勤文件分析：去指定部门下载考勤文件，对考勤内容做解析入库

三、业务场景简单示意图：

 

四、简单拆分

    产品DataInfo：生产者生产给消费者所需要的数据结构

    工厂FtpFactory：产品数据交互

    消费者ADPT、BDPT、CDPT、DDPT、EDPT：a、b、c、d、e多个同时消费的消费者

    生产者DownloadFile：生产消费者需要的数据结构

    功能入口LogPaser：业务初始化代码开始执行的位置

五、简单实现

    1.添加maven项目依赖

<dependency>
    <groupId>com.lmax</groupId>
    <artifactId>disruptor</artifactId>
    <version>3.2.1</version>
</dependency>
<dependency>
    <groupId>org.projectlombok</groupId>
    <artifactId>lombok</artifactId>
</dependency>

    2.定义产品(DataInfo)

import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;

import java.util.List;
import java.util.Map;

/**
 * @Description
 * @ClassName DataInfo
 * @Author Morik
 * @date 2020.09.28 11:24
 */
@Slf4j
@Setter
@Getter
public class DataInfo {
    Map<String, List<FtpInfo>> datainfo;
    @Setter
    @Getter
    class FtpInfo {
        String url;
        String uri;
        String name;
        int delay;
        String dartment;
    }
}

3.创建工厂(FtpFactory)

import com.lmax.disruptor.EventFactory;

/**
 * @Description
 * @ClassName FtpFactory
 * @Author Morik
 * @date 2020.09.28 11:31
 */
public class FtpFactory implements EventFactory<DataInfo> {

    @Override
    public DataInfo newInstance() {
        return new DataInfo();
    }
}

4.定义消费者(以ADPT为例，多个消费者结构都一样)

import com.lmax.disruptor.EventHandler;
import lombok.extern.slf4j.Slf4j;

import java.util.List;
import java.util.Map;

/**
 * @Description
 * @ClassName ADPT
 * @Author Morik
 * @date 2020.09.28 11:30
 */
@Slf4j
public class ADPT implements EventHandler<DataInfo> {

    @Override
    public void onEvent(DataInfo dataInfo, long l, boolean b) throws Exception {
        Map<String, List<DataInfo.FtpInfo>> data = dataInfo.getDatainfo();
        if (data != null && !data.isEmpty()) {
            List<DataInfo.FtpInfo> person = data.get("销售部");
            if (person != null && !person.isEmpty()) {
                for (DataInfo.FtpInfo df : person) {
                    log.info("部门:" + df.dartment + "姓名:" + df.name + "的考勤已解析入库完成");
                }
            }
        }
    }
}

5.定义生产者(DownloadFile)

import com.lmax.disruptor.RingBuffer;
import lombok.extern.slf4j.Slf4j;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;

/**
 * @Description
 * @ClassName DownloadFile
 * @Author Morik
 * @date 2020.09.28 11:45
 */
@Slf4j
public class DownloadFile {
    RingBuffer<DataInfo> ringBuffer;

    public DownloadFile(RingBuffer<DataInfo> ringBuffer) {
        this.ringBuffer = ringBuffer;
    }

    public void downloading() {
        long sequence = ringBuffer.next();
        try {
            //取出空队列
            DataInfo df = ringBuffer.get(sequence);
            //获取时间队列传递数据
            df.setDatainfo(downlod());
        } finally {
            //发布事件
            ringBuffer.publish(sequence);
        }
    }

    volatile List<DataInfo.FtpInfo> ftps = initData();

    //ftp下载任务分发
    private Map<String, List<DataInfo.FtpInfo>> downlod() {
        //创建线程池
        int N_CONSUMERS = Runtime.getRuntime().availableProcessors();
        ExecutorService executorService = Executors.newFixedThreadPool(N_CONSUMERS);
        ReentrantLock look = new ReentrantLock();
        try {
            look.lock();
            for (DataInfo.FtpInfo ftp : ftps) {
                executorService.submit(() -> {
                    //ftp具体下载函数
                    try {
                        log.info("部门名称：" + ftp.dartment + ftp.name + "的考勤文件正在下载");
                        Thread.sleep(ftp.delay);
                        ftp.uri = "d://" + ftp.dartment + "/" + ftp.name + ".text";
                        log.info("部门名称：" + ftp.dartment + ftp.name + "的考勤文件已下载完成");
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                });
            }
            executorService.shutdown();//只是不能再提交新任务，等待执行的任务不受影响
            //等待线程池线程全部执行完成
            try {
                boolean loop = true;
                do {    //等待所有任务完成
                    loop = !executorService.awaitTermination(5, TimeUnit.SECONDS);  //阻塞，直到线程池里所有任务结束
                } while (loop);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            log.info("考勤文件总个数：" + ftps.size() + "已下载完成");
            Map<String, List<DataInfo.FtpInfo>> map =
                    ftps.stream().collect(Collectors.groupingBy(DataInfo.FtpInfo::getDartment));
            log.info("按部门分组：" + map.size() + "个部门考勤文件下载完成");
            return map;
        } finally {
            look.unlock();
        }

    }

    //伪造数据
    private List<DataInfo.FtpInfo> initData() {
        List<DataInfo.FtpInfo> ftps = new ArrayList<>();
        DataInfo.FtpInfo f01 = new DataInfo().new FtpInfo();
        f01.name = "张三";
        f01.dartment = "财务部";
        f01.delay = 5689;
        ftps.add(f01);

        DataInfo.FtpInfo f02 = new DataInfo().new FtpInfo();
        f02.name = "李四";
        f02.dartment = "行政部";
        f02.delay = 6583;
        ftps.add(f02);

        DataInfo.FtpInfo f03 = new DataInfo().new FtpInfo();
        f03.name = "王五";
        f03.dartment = "人事部";
        f03.delay = 7689;
        ftps.add(f03);

        DataInfo.FtpInfo f04 = new DataInfo().new FtpInfo();
        f04.name = "李六";
        f04.dartment = "技术部";
        f04.delay = 7889;
        ftps.add(f04);

        DataInfo.FtpInfo f05 = new DataInfo().new FtpInfo();
        f05.name = "李七";
        f05.dartment = "技术部";
        f05.delay = 7882;
        ftps.add(f05);

        DataInfo.FtpInfo f06 = new DataInfo().new FtpInfo();
        f06.name = "李八";
        f06.dartment = "技术部";
        f06.delay = 7882;
        ftps.add(f06);

        DataInfo.FtpInfo f07 = new DataInfo().new FtpInfo();
        f07.name = "李九";
        f07.dartment = "技术部";
        f07.delay = 7182;
        ftps.add(f07);


        DataInfo.FtpInfo f010 = new DataInfo().new FtpInfo();
        f010.name = "赵毅";
        f010.dartment = "技术部";
        f010.delay = 7182;
        ftps.add(f010);

        DataInfo.FtpInfo f011 = new DataInfo().new FtpInfo();
        f011.name = "赵二";
        f011.dartment = "销售部";
        f011.delay = 7582;
        ftps.add(f011);

        DataInfo.FtpInfo f012 = new DataInfo().new FtpInfo();
        f012.name = "赵三";
        f012.dartment = "销售部";
        f012.delay = 7152;
        ftps.add(f012);

        DataInfo.FtpInfo f013 = new DataInfo().new FtpInfo();
        f013.name = "赵四";
        f013.dartment = "销售部";
        f013.delay = 7142;
        ftps.add(f013);

        DataInfo.FtpInfo f014 = new DataInfo().new FtpInfo();
        f014.name = "赵五";
        f014.dartment = "销售部";
        f014.delay = 7132;
        ftps.add(f014);

        return ftps;
    }
}

6.构建业务入口(LogPaser)

import com.lmax.disruptor.EventFactory;
import com.lmax.disruptor.RingBuffer;
import com.lmax.disruptor.YieldingWaitStrategy;
import com.lmax.disruptor.dsl.Disruptor;
import com.lmax.disruptor.dsl.ProducerType;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * @Description
 * @ClassName LogPaser
 * @Author Morik
 * @date 2020.09.28 12:31
 */

@Service
@EnableScheduling
public class LogPaser {
    @Scheduled(fixedRate = 30000)  //30s执行一次
    public void downlogs() {
        ExecutorService executor = Executors.newCachedThreadPool();
        EventFactory<DataInfo> eventFactory = new FtpFactory();
        int ringBufferSize = 2;
        Disruptor<DataInfo> disruptor =
                new Disruptor(eventFactory, ringBufferSize, executor, ProducerType.SINGLE, new YieldingWaitStrategy());
        disruptor.handleEventsWith(new ADPT(), new BDPT(), new CDPT(), new DDPT(), new EDPT());
        disruptor.start();
        RingBuffer<DataInfo> ringBuffer = disruptor.getRingBuffer();
        DownloadFile ftpDownLoding = new DownloadFile(ringBuffer);
        ftpDownLoding.downloading();
        disruptor.shutdown();
        executor.shutdown();
    }
}

7.运行结果

8.备注说明：

8.1、生产者：

        ------>ReentrantLock 可重入锁:一般不推荐直接使用synchronized，因为频繁的资源竞争会导致锁升级

       ------->volatile:在多任务同时修改的情况下要，暴露可见性保证及时有效的进行版本值修改的同时防止jvm优化指令重排

8.2、等待线程池任务全部被执行完成后再进行主业务流程执行还可以用CountDownLatch：

      ------->设置原子计数器大小为总任务量：CountDownLatch latch = new CountDownLatch(list.size());

     ------->在任务循环提交中加入计数减一函数：latch .countDown();

    ------->在需要等待的地方调用关闭任务提交和任务挂起函数：executorService.shutdown();latch.await();