基于Flux的动态批量运行任务（异步版）

Github源码

相关文章

基于Flux的动态批量运行任务（阻塞版）
基于Flux的动态批量运行任务（异步版）

应用场景

事先不确定数据的数目，但是需要并发分批处理数据（比如，查询数据库时，可边读取数据，边异步分批处理数据）。故实现基于Flux的动态批量运行任务。

调度器：决定任务在多线程、单线程还是当前线程执行。

代码实现

Maven引用Flux

        <dependency>
            <groupId>io.projectreactor</groupId>
            <artifactId>reactor-core</artifactId>
            <version>3.3.0.RELEASE</version>
        </dependency>

import reactor.core.publisher.Mono;
import reactor.core.scheduler.Scheduler;
import reactor.core.scheduler.Schedulers;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Objects;
import java.util.function.Consumer;

/**
 * 基于Flux的动态批量运行任务（异步版）
 *
 * @param <T> the type parameter
 */
public class FluxBatchRunner<T> {
    private Scheduler scheduler;
    private int batchSize;
    private Consumer<Collection<T>> consumer;
    private Collection<T> records;

    public FluxBatchRunner(int batchSize, Consumer<Collection<T>> consumer) {
        if (batchSize <= 0) {
            throw new IllegalArgumentException();
        }
        Objects.requireNonNull(consumer);
        this.batchSize = batchSize;
        this.consumer = consumer;
        scheduler = Schedulers.parallel();
        records = new ArrayList<>(batchSize);
    }

    public FluxBatchRunner<T> setBatchSize(int batchSize) {
        this.batchSize = batchSize;
        return this;
    }

    public FluxBatchRunner<T> setScheduler(Scheduler scheduler) {
        this.scheduler = scheduler;
        return this;
    }

    /**
     * 添加数据
     *
     * @param t the t
     */
    public void add(T t) {
        records.add(t);
        if (records.size() == batchSize) {
            subscribeToMono(records);
            // reset
            records = new ArrayList<>(batchSize);
        }
    }

    /**
     * Do final.
     */
    public void doFinal() {
        if (!records.isEmpty()) {
            subscribeToMono(this.records);
        }
    }

    private void subscribeToMono(Collection<T> records) {
        Mono.just(records).subscribeOn(scheduler).subscribe(data -> consumer.accept(data));
    }
}

使用示例

public class FluxBatchRunnerTest {

    @Test
    public void doFinal() throws InterruptedException {
        CountDownLatch countDownLatch = new CountDownLatch(5);
        FluxBatchRunner<Integer> batchRunner = new FluxBatchRunner<>(20, data -> {
            System.out.printf("begin:: thread=%s; data=%s%n", Thread.currentThread().getName(), data.toString());
            try {
                Thread.sleep(200);
                countDownLatch.countDown();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            System.out.printf("end:: thread=%s; size=%s%n", Thread.currentThread().getName(), data.size());
        });
        IntStream.range(0, 100).forEach(batchRunner::add);
        batchRunner.doFinal();
        countDownLatch.await();
    }
}

结果