Java 实现多线程Queue异步队列处理任务

Java 实现多线程Queue异步队列处理任务

开发任务中，对于一些业务类型可进行异步处理，不使用rpc,mq，@Async注解，一样异步处理操作
利用多线程，将多个数据推送到队列中，进行异步批量的处理

配置队列定义

/**
 * 异步队列初始方法
 * @author
 * @date 2023-05-11 16:30
 **/
public interface ConsumptionT<T> {
    void start(String var1, HandleRunT<T> var2);

    void stop();

    void push(T var1) throws Exception;
}

/**
 * @author 
 * @date 2023-05-11 16:31
 **/
public interface HandleRunT<T> {
    void doRun(List<T> var1);
}

import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;

/**
 * @author 
 * @date 2023-05-11 16:31
 **/

public class QueueAndRunT<T> implements ConsumptionT<T> {
    final int SAVE_NUM = 30000;
    LinkedBlockingQueue<T> queueList;
    private String queueName;
    private HandleRunT<T> runnable;
    private Thread checkThread;
    private List<Thread> threadList = new LinkedList();
    private int runCount = 5;
    private int runPop = 40;
    private int runMaxCount = 20;
    private boolean isRun = true;
    private final int ONE_MIN = 600000;
    private int waitTime = 300;
    
    public QueueAndRunT() {
    }
    
    public void setWaitTime(int time) {
        if (time > 0) {
            this.waitTime = time;
        }

    }

    public QueueAndRunT(int count, int pop, int maxCount) {
        if (count > 0) {
            this.runCount = count;
        }
        if (pop > 0) {
            this.runPop = pop;
        }
        if (maxCount > 0) {
            this.runMaxCount = maxCount;
        }
    }
    
    public void start(String name, HandleRunT<T> c, int saveNum) {
        this.queueList = new LinkedBlockingQueue(saveNum);
        this.queueName = name;
        this.runnable = c;

        for (int i = 0; i < this.runCount; ++i) {
            Thread thread = new Thread(new Runnable() {
                @Override
                public void run() {
                    while (QueueAndRunT.this.isRun) {
                        try {
                            QueueAndRunT.this.doRun();
                        } catch (Exception var2) {
                            System.err.println(QueueAndRunT.this.queueName + "运行错误:" + var2);
                        }
                    }

                }
            });
            thread.start();
            this.threadList.add(thread);
        }
        
        this.checkThread = new Thread(new Runnable() {
            @Override
            public void run() {
                int wait = 0;

                while (QueueAndRunT.this.isRun) {
                    try {
                        Thread.sleep((long) (600000 - wait));
                        wait = QueueAndRunT.this.checkRunSystem();
                    } catch (InterruptedException var3) {
                    }
                }

            }
        });
        this.checkThread.start();
    }
    
    private int checkRunSystem() {
        long t1 = System.currentTimeMillis();
        int size;
        if (this.queueList != null) {
            size = this.queueList.size();
            int value = size + 1000;
            if (value >= 30000 && this.runCount < this.runMaxCount) {
                int capacity = this.runCount / 2;
                if (capacity > 0) {
                    for (int i = 0; i < capacity; ++i) {
                        Thread thread = new Thread(new Runnable() {
                            public void run() {
                                while (QueueAndRunT.this.isRun) {
                                    try {
                                        QueueAndRunT.this.doRun();
                                    } catch (Exception var2) {
                                    }
                                }

                            }
                        });
                        thread.start();
                        this.threadList.add(thread);
                    }

                    this.runCount += capacity;
                }
            }
        }
        size = this.threadList.size();
        System.err.println(this.queueName + "运行的线程数为:" + size);
        return (int) (System.currentTimeMillis() - t1);
    }
    private void doRun() throws Exception {
        List<T> readList = new LinkedList();

        for (int i = 0; i < this.runPop; ++i) {
            T poll = this.queueList.poll();
            if (poll == null) {
                break;
            }

            readList.add(poll);
        }

        if (!readList.isEmpty()) {
            this.runnable.doRun(readList);
        } else {
            Thread.sleep((long)this.waitTime);
        }

    }
    @Override
    public void start(String name, HandleRunT<T> c) {
        this.start(name, c, 30000);
    }

    @Override
    public void stop() {
        this.isRun = false;
        if (this.threadList != null && !this.threadList.isEmpty()) {
            Iterator var1 = this.threadList.iterator();

            while (var1.hasNext()) {
                Thread thread = (Thread) var1.next();
                thread.interrupt();
            }
        }

    }
    @Override
    public void push(T t) throws Exception {
        if (t != null) {
            this.queueList.put(t);
        }

    }
}

队列启动初始化

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.CommandLineRunner;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;

import javax.annotation.PreDestroy;

/**
 * 应用队列初始化
 *
 * @author
 * @date 2021-10-20 10:04
 **/
@Order(Integer.MIN_VALUE)
@Slf4j
@Component
@RequiredArgsConstructor
public class QueueAction implements CommandLineRunner {
private final TextQueue textQueue;

    @Override
    public void run(String... args) throws Exception {
        textQueue.start();
        log.info("队列初始化成功");
    }

    /**
     * 被销毁使用的方法
     */
    @PreDestroy
    public void destroy() {
        textQueue.stop();
    }

}

实际应用队列

import com.example.demo.queue.conf.HandleRunT;
import com.example.demo.queue.conf.QueueAndRunT;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;

import java.util.List;

/**
 * @author 
 * @date 2022-01-04
 **/
@Slf4j
@Component
@RequiredArgsConstructor
public class TextQueue {


    private QueueAndRunT<String> queueAndRunT;

    private String name = "textQueue";
    public void start() {
        queueAndRunT = new QueueAndRunT(5, 5000, 10);
        //聚合3秒数据，默认300毫秒
        //queueAndRunT.setWaitTime(3000);
        queueAndRunT.start(name, new HandleRunT<String>() {
            @Override
            public void doRun(List<String> list) {
                try {
                    System.out.println(list.size());
                    for (String cardImportDTO : list) {
                        System.out.println(cardImportDTO);
                    }
                } catch (Exception e) {
                    log.info("处理失败：{}", e.getMessage());
                }
            }
        });
    }
    public void stop() {
        if (queueAndRunT != null) {
            queueAndRunT.stop();
        }
    }

    public void push(String simInfo) {
        try {
            queueAndRunT.push(simInfo);
        } catch (Exception e) {
            log.warn("push错误:" + e.getLocalizedMessage());
        }
    }

}

@Slf4j
@RestController
@RequiredArgsConstructor
@RequestMapping(value = "/test", produces = MediaType.APPLICATION_JSON_VALUE)
public class TestController {

    private final TextQueue textQueue;

    /**
     *	测试
     * @param str
     */
    @GetMapping("/test")
    public void getById(@RequestParam("str") String str) {
        for (int i = 0; i < 5000; i++) {
        	//push进去的都是异步处理的
            textQueue.push(str + i);
        }
    }
}