resque java_聊聊jesque的WorkerImpl与WorkerPool

序

本文主要讲一下jesque的WorkerImpl与WorkerPool。

resque

Resque是一个使用redis来创建后台任务的ruby组件。而jesque是其java版本。通常用来做延时队列。

WorkerImpl

List queues = Arrays.asList(delayedQueue);

final Worker worker = new WorkerImpl(jesqueConfig,queues, new MapBasedJobFactory(map(entry("DemoJob", DemoJob.class))));

final Thread workerThread = new Thread(worker);

workerThread.start();

这是worker实例

jesque-2.1.2-sources.jar!/net/greghaines/jesque/worker/WorkerImpl.java

/**

* Starts this worker. Registers the worker in Redis and begins polling the queues for jobs.

* Stop this worker by calling end() on any thread.

*/

@Override

public void run() {

if (this.state.compareAndSet(NEW, RUNNING)) {

try {

renameThread("RUNNING");

this.threadRef.set(Thread.currentThread());

this.jedis.sadd(key(WORKERS), this.name);

this.jedis.set(key(WORKER, this.name, STARTED), new SimpleDateFormat(DATE_FORMAT).format(new Date()));

this.listenerDelegate.fireEvent(WORKER_START, this, null, null, null, null, null);

this.popScriptHash.set(this.jedis.scriptLoad(ScriptUtils.readScript(POP_LUA)));

this.lpoplpushScriptHash.set(this.jedis.scriptLoad(ScriptUtils.readScript(LPOPLPUSH_LUA)));

this.multiPriorityQueuesScriptHash

.set(this.jedis.scriptLoad(ScriptUtils.readScript(POP_FROM_MULTIPLE_PRIO_QUEUES)));

poll();

} catch (Exception ex) {

LOG.error("Uncaught exception in worker run-loop!", ex);

this.listenerDelegate.fireEvent(WORKER_ERROR, this, null, null, null, null, ex);

} finally {

renameThread("STOPPING");

this.listenerDelegate.fireEvent(WORKER_STOP, this, null, null, null, null, null);

this.jedis.srem(key(WORKERS), this.name);

this.jedis.del(key(WORKER, this.name), key(WORKER, this.name, STARTED), key(STAT, FAILED, this.name),

key(STAT, PROCESSED, this.name));

this.jedis.quit();

this.threadRef.set(null);

}

} else if (RUNNING.equals(this.state.get())) {

throw new IllegalStateException("This WorkerImpl is already running");

} else {

throw new IllegalStateException("This WorkerImpl is shutdown");

}

}

实现了runnable方法，里头poll方法无限循环

/**

* Polls the queues for jobs and executes them.

*/

protected void poll() {

int missCount = 0;

String curQueue = null;

while (RUNNING.equals(this.state.get())) {

try {

if (threadNameChangingEnabled) {

renameThread("Waiting for " + JesqueUtils.join(",", this.queueNames));

}

curQueue = getNextQueue();

if (curQueue != null) {

checkPaused();

// Might have been waiting in poll()/checkPaused() for a while

if (RUNNING.equals(this.state.get())) {

this.listenerDelegate.fireEvent(WORKER_POLL, this, curQueue, null, null, null, null);

final String payload = pop(curQueue);

if (payload != null) {

process(ObjectMapperFactory.get().readValue(payload, Job.class), curQueue);

missCount = 0;

} else {

missCount++;

if (shouldSleep(missCount) && RUNNING.equals(this.state.get())) {

// Keeps worker from busy-spinning on empty queues

missCount = 0;

Thread.sleep(EMPTY_QUEUE_SLEEP_TIME);

}

}

}

}

} catch (InterruptedException ie) {

if (!isShutdown()) {

recoverFromException(curQueue, ie);

}

} catch (JsonParseException | JsonMappingException e) {

// If the job JSON is not deserializable, we never want to submit it again...

removeInFlight(curQueue);

recoverFromException(curQueue, e);

} catch (Exception e) {

recoverFromException(curQueue, e);

}

}

}

不断地pop和process

/**

* Materializes and executes the given job.

*

* @param job the Job to process

* @param curQueue the queue the payload came from

*/

protected void process(final Job job, final String curQueue) {

try {

this.processingJob.set(true);

if (threadNameChangingEnabled) {

renameThread("Processing " + curQueue + " since " + System.currentTimeMillis());

}

this.listenerDelegate.fireEvent(JOB_PROCESS, this, curQueue, job, null, null, null);

this.jedis.set(key(WORKER, this.name), statusMsg(curQueue, job));

final Object instance = this.jobFactory.materializeJob(job);

final Object result = execute(job, curQueue, instance);

success(job, instance, result, curQueue);

} catch (Throwable thrwbl) {

failure(thrwbl, job, curQueue);

} finally {

removeInFlight(curQueue);

this.jedis.del(key(WORKER, this.name));

this.processingJob.set(false);

}

}

而process这个方法，就是实例化目标job，然后execute

/**

* Executes the given job.

*

* @param job the job to execute

* @param curQueue the queue the job came from

* @param instance the materialized job

* @throws Exception if the instance is a {@link Callable} and throws an exception

* @return result of the execution

*/

protected Object execute(final Job job, final String curQueue, final Object instance) throws Exception {

if (instance instanceof WorkerAware) {

((WorkerAware) instance).setWorker(this);

}

this.listenerDelegate.fireEvent(JOB_EXECUTE, this, curQueue, job, instance, null, null);

final Object result;

if (instance instanceof Callable) {

result = ((Callable>) instance).call(); // The job is executing!

} else if (instance instanceof Runnable) {

((Runnable) instance).run(); // The job is executing!

result = null;

} else { // Should never happen since we're testing the class earlier

throw new ClassCastException(

"Instance must be a Runnable or a Callable: " + instance.getClass().getName() + " - " + instance);

}

return result;

}

而execute就是调用call或者run方法。

从这里可以看出是单线程阻塞的，如果一个job比较耗时，是会影响其他job的触发和执行。

WorkerPool

jesque-2.1.2-sources.jar!/net/greghaines/jesque/worker/WorkerPool.java

/**

* Create a WorkerPool with the given number of Workers and the given ThreadFactory.

* @param workerFactory a Callable that returns an implementation of Worker

* @param numWorkers the number of Workers to create

* @param threadFactory the factory to create pre-configured Threads

*/

public WorkerPool(final Callable extends Worker> workerFactory, final int numWorkers,

final ThreadFactory threadFactory) {

this.workers = new ArrayList<>(numWorkers);

this.threads = new ArrayList<>(numWorkers);

this.eventEmitter = new WorkerPoolEventEmitter(this.workers);

for (int i = 0; i < numWorkers; i++) {

try {

final Worker worker = workerFactory.call();

this.workers.add(worker);

this.threads.add(threadFactory.newThread(worker));

} catch (RuntimeException re) {

throw re;

} catch (Exception e) {

throw new RuntimeException(e);

}

}

}

/**

* {@inheritDoc}

*/

@Override

public void run() {

for (final Thread thread : this.threads) {

thread.start();

}

Thread.yield();

}

/**

* {@inheritDoc}

*/

@Override

public void end(final boolean now) {

for (final Worker worker : this.workers) {

worker.end(now);

}

}

workerpool维护了一组worker实例，起线程池的作用，尽可能提高job的并发度。

doc