在上篇文章中,已经写出为什么disruptor性能为何如此高效,下面我们将对disruptor的多生产者模式的源码进行分析
先写个多生产者模式的demo
pom.xml
<dependency>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
<version>3.4.3</version>
</dependency>
写一个Event类
public class OrderEvent {
private String id;
private String name;
private long value;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public long getValue() {
return value;
}
public void setValue(long value) {
this.value = value;
}
}
写一个消费者类
import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import com.lmax.disruptor.WorkHandler;
public class Consumer implements WorkHandler<OrderEvent>{
private static AtomicInteger count = new AtomicInteger(0);
private String cid;
private Random random = new Random();
public Consumer(String cid) {
this.cid = cid;
}
@Override
public void onEvent(OrderEvent event) throws Exception {
Thread.sleep(1 * random.nextInt(5));
System.err.println("当前消费者: " + this.cid + ", 消费信息ID: " + event.getId());
count.incrementAndGet();
}
public int getCount(){
return count.get();
}
}
写一个生产者类
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import com.lmax.disruptor.RingBuffer;
public class Producer implements Runnable{
private RingBuffer<OrderEvent> ringBuffer;
private CountDownLatch latch;
public Producer(RingBuffer<OrderEvent> ringBuffer, CountDownLatch latch) {
this.ringBuffer = ringBuffer;
this.latch = latch;
}
@Override
public void run() {
try {
latch.await();
} catch (InterruptedException e) {
e.printStackTrace();
}
for (int i=0; i<2; i++) {
this.sendData(UUID.randomUUID().toString());
}
}
private void sendData(String uuid) {
long sequence = ringBuffer.next();
try {
OrderEvent orderEvent = ringBuffer.get(sequence);
orderEvent.setId(uuid);
} finally {
ringBuffer.publish(sequence);
}
}
}
写一个main类
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import com.lmax.disruptor.EventFactory;
import com.lmax.disruptor.ExceptionHandler;
import com.lmax.disruptor.RingBuffer;
import com.lmax.disruptor.SequenceBarrier;
import com.lmax.disruptor.WorkerPool;
import com.lmax.disruptor.YieldingWaitStrategy;
import com.lmax.disruptor.dsl.ProducerType;
public class Main {
public static void main(String[] args) throws Exception {
//1 创建RingBuffer
RingBuffer<OrderEvent> ringBuffer = RingBuffer.create(
ProducerType.MULTI,
new EventFactory<OrderEvent>() {
@Override
public OrderEvent newInstance() {
return new OrderEvent();
}
},
1024 * 1024,
new BlockingWaitStrategy());
//2 通过Ringbuffer创建一个屏障
SequenceBarrier sequenceBarrier = ringBuffer.newBarrier();
//3 创建多个消费者
Consumer[] consumers = new Consumer[10];
for (int i=0; i<10; i++) {
consumers[i] = new Consumer("C" + i);
}
//4 构建多消费者工作池
WorkerPool<OrderEvent> workerPool = new WorkerPool<OrderEvent>(
ringBuffer,
sequenceBarrier,
new EventExceptionHandler(),
consumers);
//5 设置多个消费者的sequence序号 用于单独统计消费进度, 并且设置到ringbuffer中
ringBuffer.addGatingSequences(workerPool.getWorkerSequences());
//6 启动
ExecutorService es = Executors.newFixedThreadPool(5);
workerPool.start(es);
final CountDownLatch latch = new CountDownLatch(1);
//7 生产者产生数据
for (int i=0; i<10; i++) {
new Thread(new Producer(ringBuffer, latch)).start();
}
System.err.println("----------线程创建完毕,开始生产数据----------");
latch.countDown();
Thread.sleep(10000);
System.err.println("任务总数:" + consumers[2].getCount());
}
static class EventExceptionHandler implements ExceptionHandler<OrderEvent> {
public void handleEventException(Throwable ex, long sequence, OrderEvent event) {
}
public void handleOnStartException(Throwable ex) {
}
public void handleOnShutdownException(Throwable ex) {
}
}
}
运行结构如下:
----------线程创建完毕,开始生产数据----------
当前消费者: C2, 消费信息ID: be91ab0d-1960-4d89-a466-03d018685d96
当前消费者: C1, 消费信息ID: 60297cb1-6198-4d6e-b6af-3b7bdbd5377c
当前消费者: C1, 消费信息ID: 8976cc2b-e758-406e-a186-a164e724d1af
当前消费者: C4, 消费信息ID: 48fb444e-3e79-4af5-803a-4721f70298d8
当前消费者: C2, 消费信息ID: 10a25687-8961-4b93-baf1-d6e4d30588e8
当前消费者: C4, 消费信息ID: 79087338-430f-4cc6-93b7-cb0f57dc065d
当前消费者: C0, 消费信息ID: 85c6addd-cf2f-4a20-9cf0-64239bc74ec7
当前消费者: C2, 消费信息ID: 2208defc-14b0-481d-877e-cfa61aaefc09
当前消费者: C2, 消费信息ID: 641a19f1-0e38-4ebe-b59b-1c51d59666a1
当前消费者: C3, 消费信息ID: d90de8b3-cfb9-47b7-81c4-be1974dd7ec3
当前消费者: C2, 消费信息ID: 26047e6b-2b3f-4c33-b0bd-d43b99e5a709
当前消费者: C1, 消费信息ID: 74ac73c4-8c3d-4d43-9653-a3b05798fe0e
当前消费者: C2, 消费信息ID: 08ba6759-688d-404c-95f3-4bca5b2508f3
当前消费者: C2, 消费信息ID: 61e66e31-33e1-4b2b-ba58-005754c98347
当前消费者: C0, 消费信息ID: 96d1f9af-e0bd-4da6-864c-f58d855d8fa3
当前消费者: C4, 消费信息ID: 6385aee1-9544-4488-aa2a-c8f7afdb9ddb
当前消费者: C3, 消费信息ID: ea35dfcc-71a8-4524-b614-39ea5a7554cb
当前消费者: C2, 消费信息ID: 43c38d4f-e088-4343-b12a-d9f77eeba439
当前消费者: C1, 消费信息ID: 11b9d160-7167-4dd5-88b7-e83abfc8868e
当前消费者: C0, 消费信息ID: e42366cd-cfc6-4a89-9c4a-907ba99f490e
任务总数:20
下面对main方法进行源码解析:
1 创建RingBuffer
指定ProductType为Multi,和单生产者区别在于Sequencer 为MultiProducerSequencer
在实例化MultiProducerSequencer时,主要做以下几件事:
public MultiProducerSequencer(int bufferSize, final WaitStrategy waitStrategy)
{
super(bufferSize, waitStrategy);
availableBuffer = new int[bufferSize];
indexMask = bufferSize - 1;
indexShift = Util.log2(bufferSize);//bufferSize = 2 ^ n == > indexShift=n
initialiseAvailableBuffer();//availableBuffer 数组初始值都是-1
}
2 通过Ringbuffer创建一个屏障(这个和单生产者一样,都是通过父类AbstractSequencer去创建ProcessingSequenceBarrier)
SequenceBarrier sequenceBarrier = ringBuffer.newBarrier();
3.构建多消费者工作池
public WorkerPool(
final RingBuffer<T> ringBuffer,
final SequenceBarrier sequenceBarrier,
final ExceptionHandler<? super T> exceptionHandler,
final WorkHandler<? super T>... workHandlers)
{
this.ringBuffer = ringBuffer;
final int numWorkers = workHandlers.length;
workProcessors = new WorkProcessor[numWorkers];
//对每个消费者 都封装到WorkProcessor,这里区别于单生产者模式(BatchEventProcessor)
//每个消费者都共用一个workSequence(序号)和SequenceBarrier(序号栅栏),但是每个消费者都有自己的(序号)
//workSequence 就相当于锁,当消费者消费之前,必须向workSequence赋值,赋值成功后,才能消费
//每个消费者都有自己的(序号)记录当前消费的序号
for (int i = 0; i < numWorkers; i++)
{
workProcessors[i] = new WorkProcessor<>(
ringBuffer,
sequenceBarrier,
workHandlers[i],
exceptionHandler,
workSequence);
}
}
4. workPool启动
public RingBuffer<T> start(final Executor executor)
{
if (!started.compareAndSet(false, true))
{
throw new IllegalStateException("WorkerPool has already been started and cannot be restarted until halted.");
}
//设置ProcessingSequenceBarrier中cursor和workSequence 值一样
final long cursor = ringBuffer.getCursor();
workSequence.set(cursor);
for (WorkProcessor<?> processor : workProcessors)
{
processor.getSequence().set(cursor);
executor.execute(processor);
}
return ringBuffer;
}
线程池执行execute实际上是WorkProcessor(单生产者是BatchEventProcessor)
public void run()
{
if (!running.compareAndSet(false, true))
{
throw new IllegalStateException("Thread is already running");
}
sequenceBarrier.clearAlert();
//通知消费前,开始事件
notifyStart();
boolean processedSequence = true;
long cachedAvailableSequence = Long.MIN_VALUE;
long nextSequence = sequence.get();
T event = null;
while (true)
{
try
{
// if previous sequence was processed - fetch the next sequence and set
// that we have successfully processed the previous sequence
// typically, this will be true
// this prevents the sequence getting too far forward if an exception
// is thrown from the WorkHandler
if (processedSequence)
{
processedSequence = false;
/**
* 下面的do while循环,就是多个消费者再向WorkSequence要“锁”
* 只不过要“锁”成功的标志时,对WorkSequence赋值成功
*/
do
{
nextSequence = workSequence.get() + 1L;
sequence.set(nextSequence - 1L);
}
while (!workSequence.compareAndSet(nextSequence - 1L, nextSequence));
}
if (cachedAvailableSequence >= nextSequence)
{
event = ringBuffer.get(nextSequence);
//消费者消费数据
workHandler.onEvent(event);
processedSequence = true;
}
else
{
//缓存最大可用序号
cachedAvailableSequence = sequenceBarrier.waitFor(nextSequence);
}
}
catch (final TimeoutException e)
{
notifyTimeout(sequence.get());
}
catch (final AlertException ex)
{
if (!running.get())
{
break;
}
}
catch (final Throwable ex)
{
// handle, mark as processed, unless the exception handler threw an exception
exceptionHandler.handleEventException(ex, nextSequence, event);
processedSequence = true;
}
}
//通知消费前,关闭事件
notifyShutdown();
running.set(false);
}
多个消费者,要想消费,必须要对WorkSequence进行赋值,这里采用了CAS
do
{
nextSequence = workSequence.get() + 1L;
sequence.set(nextSequence - 1L);
}
while (!workSequence.compareAndSet(nextSequence - 1L, nextSequence));
赋值成功后,通过等待策略去查询生产者有没有生产数据
cachedAvailableSequence = sequenceBarrier.waitFor(nextSequence);
@Override
public long waitFor(final long sequence)
throws AlertException, InterruptedException, TimeoutException
{
checkAlert();
long availableSequence = waitStrategy.waitFor(sequence, cursorSequence, dependentSequence, this);
if (availableSequence < sequence)
{
return availableSequence;
}
return sequencer.getHighestPublishedSequence(sequence, availableSequence);
}
sequencer.getHighestPublishedSequence(sequence, availableSequence);
com.lmax.disruptor.MultiProducerSequencer.getHighestPublishedSequence(long, long)
这里和单生产者不同(直接返回availableSequence)
@Override
public long getHighestPublishedSequence(long lowerBound, long availableSequence)
{
for (long sequence = lowerBound; sequence <= availableSequence; sequence++)
{
//判断sequence是否可用
if (!isAvailable(sequence))
{
return sequence - 1;
}
}
return availableSequence;
}
@Override
public boolean isAvailable(long sequence)
{
int index = calculateIndex(sequence);//计算索引值
int flag = calculateAvailabilityFlag(sequence);//计算sequence,在槽中被使用的次数
long bufferAddress = (index * SCALE) + BASE;
//然后availableBuffer 记录的值,进行比较
return UNSAFE.getIntVolatile(availableBuffer, bufferAddress) == flag;
}
4. 生产数据
private void sendData(String uuid) {
long sequence = ringBuffer.next();
try {
OrderEvent orderEvent = ringBuffer.get(sequence);
orderEvent.setId(uuid);
} finally {
ringBuffer.publish(sequence);
}
}
4.1 生产sequence
long sequence = ringBuffer.next();
com.lmax.disruptor.MultiProducerSequencer.next(int)
public long next(int n)
{
if (n < 1 || n > bufferSize)
{
throw new IllegalArgumentException("n must be > 0 and < bufferSize");
}
long current;
long next;
do
{
current = cursor.get();
next = current + n;
long wrapPoint = next - bufferSize;
//因为是多生产者,所有缓存sequence,必须要用成员变量了
long cachedGatingSequence = gatingSequenceCache.get();
// 下面的操作和单生产者差不多,只是缓存数据方式不一样
if (wrapPoint > cachedGatingSequence || cachedGatingSequence > current)
{
long gatingSequence = Util.getMinimumSequence(gatingSequences, current);
if (wrapPoint > gatingSequence)
{
LockSupport.parkNanos(1); // TODO, should we spin based on the wait strategy?
continue;
}
gatingSequenceCache.set(gatingSequence);
}
// 这个地方和单生产者不一样,这个地方一旦获取成功,就更新当前所有生产者的sequence
// ,更新成功就返回,更新不成功就重新获取
else if (cursor.compareAndSet(current, next))
{
break;
}
}
while (true);
return next;
}
4.2 发布event
public void publish(final long sequence)
{
setAvailable(sequence);//设置sequence,为可用
waitStrategy.signalAllWhenBlocking();//唤醒消费者,可以消费
}
setAvailable(sequence);
发布某个sequence之前的都可以被消费了需要将availableBuffer上对应sequence下标的值设置为第几次用到这个槽
private void setAvailable(final long sequence)
{
setAvailableBufferValue(calculateIndex(sequence), calculateAvailabilityFlag(sequence));
}
private void setAvailableBufferValue(int index, int flag)
{
long bufferAddress = (index * SCALE) + BASE;
UNSAFE.putOrderedInt(availableBuffer, bufferAddress, flag);
}
这个地方赋值,是为了com.lmax.disruptor.MultiProducerSequencer.getHighestPublishedSequence(long, long)做准备
当消费者获取最大可用sequence,要进行一次判断。
这样整个多生产者模式就结束了。