通常生产者-消费者的经典实现方式是,启动一个消费者线程从阻塞队列里获取消息进行消费,(多个)生产者往队列里插入待消费的数据然后立即返回。如果生产者生产的速率远大于消费者消费的速率,那么队列的待处理数据就会累积得越来越多。
本文的重点是封装实现(单个)消费者线程可以“批量处理”队列里的消息,这类似于多条数据插入数据库,由多次insert优化为一次batch insert。
顾名思义,“多消费者”就是开启多个消费者线程(其中每个消费者线程均可以批量处理队列消息),这里借用Java线程池来管理线程的生命周期:
首先,定义一个接口表示异步消费:
import java.util.concurrent.RejectedExecutionException;
/**
* An object that accepts elements for future consuming.
*
* @param <E> The type of element to be consumed
*/
public interface AsynchronousConsumer<E> {
/**
* Accept an element for future consuming.
*
* @param e the element to be consumed
* @return true if accepted, false otherwise
* @throws NullPointerException if the specified element is null
* @throws IllegalArgumentException if some property of this element
* prevents it from being accepted for future consuming
* @throws RejectedExecutionException if the element
* cannot be accepted for consuming
*/
public boolean accept(E e);
}
再定义一个消费者线程(由于线程托管给线程池管理了,这里是定义一个Runnable),多个Runnable通过共用BlockingQueue来实现多个消费者消费。这里 通过指定“batchSize”来表示批量处理(如果值大于1的话),TimeUnit指明了线程(Runnable)的最大空闲时间,超过该时间将会自动退出(如果值为0则表示永远等待)并提供beforeConsume、afterConsume、terminated等方法以便扩展:
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* An abstract {@code Runnable} of consumer for batches consuming Asynchronously
*
* @param <E> The type of element to be consumed
*/
public abstract class ConsumerRunnable<E> implements Runnable, AsynchronousConsumer<E>{
/**
* for recording any exception
*/
private static final Log log = LogFactory.getLog(ConsumerRunnable.class);
/**the elements queue**/
private final BlockingQueue<E> queue;
/**
* the batchSize for {@link #consume} used in every loop
*/
private volatile int batchSize;
/**
* Timeout in nanoseconds for polling an element from the
* working queue(BlockingQueue).Thread uses this timeout to
* wait up if the working queue is empty.
* zero means wait forever until an element become available.
*/
private volatile long waitTime;
/**
* If true then will cause run() quit
* when waiting for element if interrupted
*/
private volatile boolean quitIfInterrupted = true;
/**
* Allocates a new {@code ConsumerRunnable} object
* with the given initial parameters and default waitTime(0)
* which will cause the thread to wait forever until an element
* become available when the element queue is empty.
*
* @param queue the BlockingQueue for working
* @param batchSize the batch size used in {@link #consume} for every loop
*
* @throws NullPointerException if the queue is null
* @throws IllegalArgumentException if the batchSize is less then or equal to zero
*/
public ConsumerRunnable(int batchSize, BlockingQueue<E> queue){
this.batchSize = PracticalUtils.requirePositive(batchSize);
this.queue = Objects.requireNonNull(queue);
}
/**
* Allocates a new {@code ConsumerRunnable} object
* with the given initial parameters.
* A time value of zero will cause the thread to wait forever
* until an element become available when the element queue is empty.
*
* @param queue the BlockingQueue for working
* @param batchSize the batch size used in {@link #consume} for every loop
* @param time the time to wait. A time value of zero will cause
* the thread to wait forever until an element become available
* when the element queue is empty.
* @param unit the time unit of the time argument
*
* @throws NullPointerException if any specified parameter is null
* @throws IllegalArgumentException if the batchSize is less then or equal to zero,
* or the time is less than zero
*/
public ConsumerRunnable(int batchSize, long time, TimeUnit unit,
BlockingQueue<E> queue){
this.batchSize = PracticalUtils.requirePositive(batchSize);
this.queue = Objects.requireNonNull(queue);
setWaitTime(time, unit);
}
/**
* Accept an element for future consuming.
*
* <p>Default implementation is equally invoke
* {@link java.util.Queue#offer queue.offer}
*
* @param e the element to be consumed
* @throws NullPointerException if the specified element is null
* @throws IllegalArgumentException if some property of this element
* prevents it from being accepted for future consuming
* @throws RejectedExecutionException if the element
* cannot be accepted for future consuming (optional)
* @return true if accepted, false otherwise
*/
public boolean accept(E e){
Objects.requireNonNull(e);
return queue.offer(e);
}
/**
* Main worker run loop. Repeatedly gets elements from working queue
* and consumes them, while coping with a number of issues:
*
* <p>1. Each loop run is preceded by a call to beforeConsume, which
* might throw an exception, in which case the {@link #consume}
* of the current loop will not be invoked.
*
* <p>2. Assuming beforeConsume completes normally, we consume the elements,
* gathering any of its thrown exceptions to send to afterConsume. Any thrown
* exception in the afterConsume conservatively causes the runnable to quit.
*
* <p>3. After {@link #consume} of the current loop completes, we call afterExecute,
* which may also throw an exception, which will cause the runnable to quit.
*
*/
@Override
public final void run() {
starting();
final List<E> list = new ArrayList<>();
try{
while(true){
try {
E info;
int bSize = batchSize;
for(int i=0;i<bSize&&list.size()<bSize&&(info=queue.poll())!=null;i++){
list.add(info);
}
if(list.isEmpty()){
try{
beforeWait();
} catch(Throwable ingnore){}
//waiting if necessary until an element becomes available
if(waitTime==0){
list.add(queue.take());
}else{
E e = queue.poll(waitTime, TimeUnit.NANOSECONDS);
if(e==null){//timeout
break;
}
list.add(e);
}
}else{
try{
beforeConsume(list);
}catch (Throwable t1) {
list.clear();//clear to be reused for next loop
log.error("The runnable["+getClass().getName()+"] which invoked beforeConsume occurred the exception:"
+t1.toString(), t1);
continue;
}
doConsume(list);
}
} catch (InterruptedException e) {
if(quitIfInterrupted){
Thread t = Thread.currentThread();
if(!t.isInterrupted()){
//restores the interrupted status
t.interrupt();
}
break;
}
} catch(Throwable t){
if(t instanceof Error){
log.error("The runnable["+getClass().getName()+"] is about to die for the reason:"+t.toString(), t);
break;
}
// else continue;
}
}
} finally{
try{
//rechcek the list
if(!list.isEmpty()){
beforeConsume(list);
doConsume(list);
}
} finally{
terminated();
}
}
}
/**
* Consuming elements and ensure that the method
* {@code afterConsume} must be invoked in the end.
* @param list the elements that will be consumed
*/
private void doConsume(List<E> list){
Throwable t = null;
try {
consume(list);
} catch (Throwable t1) {
t = t1;
throw t1;
} finally{
try {
afterConsume(list,t);
} finally{
list.clear();//clear to be reused for next loop
}
}
}
/**
* Returns the batchSize for {@link #consume} used in every loop
*
* @return the batchSize
*
* @see #setBatchSize
*/
public int getBatchSize(){
return batchSize;
}
/**
* Sets the batchSize for next loop used in {@link #consume}.
* This overrides any value set in the constructor.
*
* @param batchSize the new batchSize
* @throws IllegalArgumentException if the new batchSize is
* less than or equal to zero
* @see #getBatchSize
*/
public void setBatchSize(int batchSize){
this.batchSize = PracticalUtils.requirePositive(batchSize);
}
/**
* Returns the element queue used by this thread. Access to the
* element queue is intended primarily for debugging and monitoring.
* This queue may be in active use. Retrieving the element queue
* does not prevent queued element from being saved.
*
* @return the element queue
*/
public BlockingQueue<E> getQueue() {
return queue;
}
/**
* Sets the time limit for the thread when waiting for
* an element to become available. This overrides any value
* set in the constructor. zero means wait forever.
* Timeout will cause the runnable to quit.
*
* @param time the time to wait. A time value of zero will cause
* the thread to wait forever until an element become available
* when the element queue is empty.
* @param unit the time unit of the {@code time} argument
* @throws IllegalArgumentException if {@code time} less than zero
* @see #getWaitTime(TimeUnit)
*/
public void setWaitTime(long time, TimeUnit unit) {
long t = unit.toNanos(PracticalUtils.requireNonNegative(time));
this.waitTime = t;
}
/**
* Returns the thread waiting time, which is the time to wait up
* when the working queue is empty. zero means wait forever.
*
* @param unit the desired time unit of the result
* @return the time limit
* @see #setWaitTime(long, TimeUnit)
*/
public long getWaitTime(TimeUnit unit) {
return unit.convert(waitTime, TimeUnit.NANOSECONDS);
}
/**
* Returns true if this runnable will quit
* if interrupted when waiting for element.
* @return true if this runnable will quit
* if interrupted when waiting for element
*/
public boolean isQuitIfInterrupted(){return quitIfInterrupted;}
/**
* Set quit or not if interrupted when waiting for element.
* @param quitIfInterrupted true will cause run() quit
* if interrupted when waiting for element
*/
public void setQuitIfInterrupted(boolean quitIfInterrupted){
this.quitIfInterrupted = quitIfInterrupted;
}
/* Extension hooks */
/**
* Method invoked at the beginning of the run() method.
* <p><b>Note:</b>if this method throw any exception then
* any other method will not be invoked and cause the runnable to quit.
*/
protected void starting() { }
/**
* Method invoked prior to current thread waiting for
* an element to become available.
* Any exception thrown by this method will be ignored.
*
* <p>This implementation does nothing, but may be customized in
* subclasses. Note: To properly nest multiple overridings, subclasses
* should generally invoke {@code super.beforeWaiting} at the end of
* this method.
*
*/
protected void beforeWait() { }
/**
* Method invoked prior to invoking {@link #consume} in the
* current runnable for every loop.
*
* <p><b>Note:</b>if this method throw any exception then the
* {@link #consume} of current loop will not be invoked
*
* <p>This implementation does nothing, but may be customized in
* subclasses. Note: To properly nest multiple overridings, subclasses
* should generally invoke {@code super.beforeConsume} at the end of
* this method.
*
* @param list the elements that will be consumed
*/
protected void beforeConsume(List<E> list) { }
/**
* For subclass to implement the consume actions.
*
* @param list the elements that will be consumed
*/
protected abstract void consume(List<E> list);
/**
* Method invoked upon one loop completion of {@link #consume} invoked.
* This method is invoked by the current thread. If
* non-null, the Throwable is the uncaught {@code Error}
* that caused execution to terminate abruptly.
*
* <p>This implementation does nothing, but may be customized in
* subclasses. Note: To properly nest multiple overridings, subclasses
* should generally invoke {@code super.afterConsume} at the
* beginning of this method.
*
* @param list the elements that has consumed(or to be consumed if the exception caused termination)
* @param t the exception that caused termination, or null if
* the current loop execution completed normally
*/
protected void