前言
通过之前对ThingsBoard
相关技术栈的预览,研究对于Actor
模型的实现
定义
Actor
是一种分布式并发编程模式,旨在将资源私有化在Actor
模型中,Actor
模型间通过消息队列通信,异步串行地处理消息,以避免多线程对于共享资源的竞争
Actor
模型由三部分组成:
- state(状态)
内部私有地属性,可以理解为资源 - behavior(行为)
处理state逻辑,可以理解为方法 - MailBox(邮箱)
即接收消息的队列,用于存储接收到的消息并在空闲时处理
实现
ThingsBoard
在common/actor模块中实现了默认的Actor
系统
从接口入手研究
TbActorId
package org.thingsboard.server.actors;
import org.thingsboard.server.common.data.EntityType;
public interface TbActorId {
/**
* Returns entity type of the actor.
* May return null if the actor does not belong to any entity.
* This method is added for performance optimization.
*
*/
EntityType getEntityType();
}
Actor
标识接口
EntityType getEntityType()
获取实体类型
TbActorRef
package org.thingsboard.server.actors;
import org.thingsboard.server.common.msg.TbActorMsg;
public interface TbActorRef {
TbActorId getActorId();
void tell(TbActorMsg actorMsg);
void tellWithHighPriority(TbActorMsg actorMsg);
}
Actor
引用接口
TbActorId getActorId()
获取标识void tell(TbActorMsg actorMsg)
告知消息 TbActorMsg1void tellWithHighPriority(TbActorMsg actorMsg)
使用高优先级告知消息
TbActorCtx
package org.thingsboard.server.actors;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.msg.TbActorMsg;
import java.util.List;
import java.util.function.Predicate;
import java.util.function.Supplier;
public interface TbActorCtx extends TbActorRef {
TbActorId getSelf();
TbActorRef getParentRef();
void tell(TbActorId target, TbActorMsg msg);
void stop(TbActorId target);
TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier<String> dispatcher, Supplier<TbActorCreator> creator);
void broadcastToChildren(TbActorMsg msg);
void broadcastToChildrenByType(TbActorMsg msg, EntityType entityType);
void broadcastToChildren(TbActorMsg msg, Predicate<TbActorId> childFilter);
List<TbActorId> filterChildren(Predicate<TbActorId> childFilter);
}
Actor
上下文接口
TbActorId getSelf()
获取自身标识TbActorRef getParentRef()
获取父引用void tell(TbActorId target, TbActorMsg msg)
告知目标Actor
消息void stop(TbActorId target)
停止目标Actor
TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier<String> dispatcher, Supplier<TbActorCreator> creator)
获取或创建子Actor
void broadcastToChildren(TbActorMsg msg)
向子Actor
广播消息void broadcastToChildrenByType(TbActorMsg msg, EntityType entityType)
向指定实体类型的子Actor
广播消息void broadcastToChildren(TbActorMsg msg, Predicate<TbActorId> childFilter)
向符合过滤器的子Actor
广播消息List<TbActorId> filterChildren(Predicate<TbActorId> childFilter)
获取过滤后的子Actor
集合
TbActor
package org.thingsboard.server.actors;
import org.thingsboard.server.common.msg.TbActorMsg;
public interface TbActor {
boolean process(TbActorMsg msg);
TbActorRef getActorRef();
default void init(TbActorCtx ctx) throws TbActorException {
}
default void destroy() throws TbActorException {
}
default InitFailureStrategy onInitFailure(int attempt, Throwable t) {
return InitFailureStrategy.retryWithDelay(5000L * attempt);
}
default ProcessFailureStrategy onProcessFailure(Throwable t) {
if (t instanceof Error) {
return ProcessFailureStrategy.stop();
} else {
return ProcessFailureStrategy.resume();
}
}
}
Actor
实体
boolean process(TbActorMsg msg)
处理消息TbActorRef getActorRef()
获取引用void init(TbActorCtx ctx) throws TbActorException
初始化方法,默认空实现void destroy() throws TbActorException
销毁方法,默认空实现InitFailureStrategy onInitFailure(int attempt, Throwable t)
获取初始化失败策略
策略为停止或延迟(可选)重试package org.thingsboard.server.actors; import lombok.Getter; import lombok.ToString; @ToString public class InitFailureStrategy { @Getter private boolean stop; @Getter private long retryDelay; private InitFailureStrategy(boolean stop, long retryDelay) { this.stop = stop; this.retryDelay = retryDelay; } public static InitFailureStrategy retryImmediately() { return new InitFailureStrategy(false, 0); } public static InitFailureStrategy retryWithDelay(long ms) { return new InitFailureStrategy(false, ms); } public static InitFailureStrategy stop() { return new InitFailureStrategy(true, 0); } }
ProcessFailureStrategy onProcessFailure(Throwable t)
获取处理消息失败策略
策略为停止或继续package org.thingsboard.server.actors; import lombok.Getter; import lombok.ToString; @ToString public class ProcessFailureStrategy { @Getter private boolean stop; private ProcessFailureStrategy(boolean stop) { this.stop = stop; } public static ProcessFailureStrategy stop() { return new ProcessFailureStrategy(true); } public static ProcessFailureStrategy resume() { return new ProcessFailureStrategy(false); } }
TbActorCreator
package org.thingsboard.server.actors;
public interface TbActorCreator {
TbActorId createActorId();
TbActor createActor();
}
Actor
创建器
TbActorId createActorId()
创建标识TbActor createActor()
创建Actor
TbActorSystem
package org.thingsboard.server.actors;
import org.thingsboard.server.common.msg.TbActorMsg;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledExecutorService;
import java.util.function.Predicate;
public interface TbActorSystem {
ScheduledExecutorService getScheduler();
void createDispatcher(String dispatcherId, ExecutorService executor);
void destroyDispatcher(String dispatcherId);
TbActorRef getActor(TbActorId actorId);
TbActorRef createRootActor(String dispatcherId, TbActorCreator creator);
TbActorRef createChildActor(String dispatcherId, TbActorCreator creator, TbActorId parent);
void tell(TbActorId target, TbActorMsg actorMsg);
void tellWithHighPriority(TbActorId target, TbActorMsg actorMsg);
void stop(TbActorRef actorRef);
void stop(TbActorId actorId);
void stop();
void broadcastToChildren(TbActorId parent, TbActorMsg msg);
void broadcastToChildren(TbActorId parent, Predicate<TbActorId> childFilter, TbActorMsg msg);
List<TbActorId> filterChildren(TbActorId parent, Predicate<TbActorId> childFilter);
}
Actor
系统
ScheduledExecutorService getScheduler()
获取执行器void createDispatcher(String dispatcherId, ExecutorService executor)
创建调度器void destroyDispatcher(String dispatcherId)
销毁调度器TbActorRef getActor(TbActorId actorId)
获取Actor
引用TbActorRef createRootActor(String dispatcherId, TbActorCreator creator)
使用创建器创建根Actor
TbActorRef createChildActor(String dispatcherId, TbActorCreator creator, TbActorId parent)
使用创建器创建指定Actor
的子Actor
void tell(TbActorId target, TbActorMsg actorMsg)
告知目标Actor
消息void tellWithHighPriority(TbActorId target, TbActorMsg actorMsg)
使用高优先级告知目标Actor
消息void stop(TbActorRef actorRef)
停止指定Actor
void stop(TbActorId actorId)
停止指定Actor
void stop()
停止系统void broadcastToChildren(TbActorId parent, TbActorMsg msg)
向指定Actor
的子Actor
广播消息void broadcastToChildren(TbActorId parent, Predicate<TbActorId> childFilter, TbActorMsg msg)
过滤指定Actor
的子Actor
后广播消息List<TbActorId> filterChildren(TbActorId parent, Predicate<TbActorId> childFilter)
获取指定Actor
过滤后的子Actor
集合
至此可看出,Actor
系统接口的实现是继续阅读的切入点
DefaultTbActorSystem
package org.thingsboard.server.actors;
import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.msg.TbActorMsg;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@Slf4j
@Data
public class DefaultTbActorSystem implements TbActorSystem {
private final ConcurrentMap<String, Dispatcher> dispatchers = new ConcurrentHashMap<>();
private final ConcurrentMap<TbActorId, TbActorMailbox> actors = new ConcurrentHashMap<>();
private final ConcurrentMap<TbActorId, ReentrantLock> actorCreationLocks = new ConcurrentHashMap<>();
private final ConcurrentMap<TbActorId, Set<TbActorId>> parentChildMap = new ConcurrentHashMap<>();
@Getter
private final TbActorSystemSettings settings;
@Getter
private final ScheduledExecutorService scheduler;
public DefaultTbActorSystem(TbActorSystemSettings settings) {
this.settings = settings;
this.scheduler = Executors.newScheduledThreadPool(settings.getSchedulerPoolSize(), ThingsBoardThreadFactory.forName("actor-system-scheduler"));
}
@Override
public void createDispatcher(String dispatcherId, ExecutorService executor) {
Dispatcher current = dispatchers.putIfAbsent(dispatcherId, new Dispatcher(dispatcherId, executor));
if (current != null) {
throw new RuntimeException("Dispatcher with id [" + dispatcherId + "] is already registered!");
}
}
@Override
public void destroyDispatcher(String dispatcherId) {
Dispatcher dispatcher = dispatchers.remove(dispatcherId);
if (dispatcher != null) {
dispatcher.getExecutor().shutdownNow();
} else {
throw new RuntimeException("Dispatcher with id [" + dispatcherId + "] is not registered!");
}
}
@Override
public TbActorRef getActor(TbActorId actorId) {
return actors.get(actorId);
}
@Override
public TbActorRef createRootActor(String dispatcherId, TbActorCreator creator) {
return createActor(dispatcherId, creator, null);
}
@Override
public TbActorRef createChildActor(String dispatcherId, TbActorCreator creator, TbActorId parent) {
return createActor(dispatcherId, creator, parent);
}
/**
* 创建 Actor,关键方法
*/
private TbActorRef createActor(String dispatcherId, TbActorCreator creator, TbActorId parent) {
//获取调度器
Dispatcher dispatcher = dispatchers.get(dispatcherId);
if (dispatcher == null) {
log.warn("Dispatcher with id [{}] is not registered!", dispatcherId);
throw new RuntimeException("Dispatcher with id [" + dispatcherId + "] is not registered!");
}
//获取标识
TbActorId actorId = creator.createActorId();
//获取邮箱
TbActorMailbox actorMailbox = actors.get(actorId);
if (actorMailbox != null) {
log.debug("Actor with id [{}] is already registered!", actorId);
} else {
//创建邮箱
//获取 Actor 创建锁
Lock actorCreationLock = actorCreationLocks.computeIfAbsent(actorId, id -> new ReentrantLock());
//加锁
actorCreationLock.lock();
try {
//获取邮箱
actorMailbox = actors.get(actorId);
//二次验证
if (actorMailbox == null) {
log.debug("Creating actor with id [{}]!", actorId);
//创建 Actor
TbActor actor = creator.createActor();
TbActorRef parentRef = null;
if (parent != null) {
//获取父 Actor
parentRef = getActor(parent);
if (parentRef == null) {
throw new TbActorNotRegisteredException(parent, "Parent Actor with id [" + parent + "] is not registered!");
}
}
//创建邮箱
TbActorMailbox mailbox = new TbActorMailbox(this, settings, actorId, parentRef, actor, dispatcher);
//将邮箱放入集合
actors.put(actorId, mailbox);
//初始化 Actor
mailbox.initActor();
actorMailbox = mailbox;
if (parent != null) {
//将标识加入对应的父子集合
parentChildMap.computeIfAbsent(parent, id -> ConcurrentHashMap.newKeySet()).add(actorId);
}
} else {
log.debug("Actor with id [{}] is already registered!", actorId);
}
} finally {
//解锁
actorCreationLock.unlock();
//移除锁
actorCreationLocks.remove(actorId);
}
}
//返回邮箱
return actorMailbox;
}
@Override
public void tellWithHighPriority(TbActorId target, TbActorMsg actorMsg) {
tell(target, actorMsg, true);
}
@Override
public void tell(TbActorId target, TbActorMsg actorMsg) {
tell(target, actorMsg, false);
}
/**
* 告知 Actor 消息,关键方法
*/
private void tell(TbActorId target, TbActorMsg actorMsg, boolean highPriority) {
//获取邮箱
TbActorMailbox mailbox = actors.get(target);
if (mailbox == null) {
throw new TbActorNotRegisteredException(target, "Actor with id [" + target + "] is not registered!");
}
if (highPriority) {
//使用高优先级告知消息
mailbox.tellWithHighPriority(actorMsg);
} else {
//告知消息
mailbox.tell(actorMsg);
}
}
@Override
public void broadcastToChildren(TbActorId parent, TbActorMsg msg) {
broadcastToChildren(parent, id -> true, msg);
}
@Override
public void broadcastToChildren(TbActorId parent, Predicate<TbActorId> childFilter, TbActorMsg msg) {
Set<TbActorId> children = parentChildMap.get(parent);
if (children != null) {
children.stream().filter(childFilter).forEach(id -> tell(id, msg));
}
}
@Override
public List<TbActorId> filterChildren(TbActorId parent, Predicate<TbActorId> childFilter) {
Set<TbActorId> children = parentChildMap.get(parent);
if (children != null) {
return children.stream().filter(childFilter).collect(Collectors.toList());
} else {
return Collections.emptyList();
}
}
@Override
public void stop(TbActorRef actorRef) {
stop(actorRef.getActorId());
}
@Override
public void stop(TbActorId actorId) {
Set<TbActorId> children = parentChildMap.remove(actorId);
if (children != null) {
for (TbActorId child : children) {
stop(child);
}
}
TbActorMailbox mailbox = actors.remove(actorId);
if (mailbox != null) {
mailbox.destroy();
}
}
@Override
public void stop() {
dispatchers.values().forEach(dispatcher -> {
dispatcher.getExecutor().shutdown();
try {
dispatcher.getExecutor().awaitTermination(3, TimeUnit.SECONDS);
} catch (InterruptedException e) {
log.warn("[{}] Failed to stop dispatcher", dispatcher.getDispatcherId(), e);
}
});
if (scheduler != null) {
scheduler.shutdownNow();
}
actors.clear();
}
}
目标指向了TbActorMailbox
TbActorMailbox
先看一下继承关系
实现了Actor
上下文接口
package org.thingsboard.server.actors;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.msg.MsgType;
import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbActorStopReason;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Predicate;
import java.util.function.Supplier;
@Slf4j
@Data
public final class TbActorMailbox implements TbActorCtx {
private static final boolean HIGH_PRIORITY = true;
private static final boolean NORMAL_PRIORITY = false;
private static final boolean FREE = false;
private static final boolean BUSY = true;
private static final boolean NOT_READY = false;
private static final boolean READY = true;
private final TbActorSystem system;
private final TbActorSystemSettings settings;
private final TbActorId selfId;
private final TbActorRef parentRef;
private final TbActor actor;
private final Dispatcher dispatcher;
private final ConcurrentLinkedQueue<TbActorMsg> highPriorityMsgs = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<TbActorMsg> normalPriorityMsgs = new ConcurrentLinkedQueue<>();
private final AtomicBoolean busy = new AtomicBoolean(FREE);
private final AtomicBoolean ready = new AtomicBoolean(NOT_READY);
private final AtomicBoolean destroyInProgress = new AtomicBoolean();
private volatile TbActorStopReason stopReason;
/**
* 初始化 Actor
*/
public void initActor() {
//异步初始化
dispatcher.getExecutor().execute(() -> tryInit(1));
}
/**
* 尝试初始化 Actor,关键方法
*/
private void tryInit(int attempt) {
try {
log.debug("[{}] Trying to init actor, attempt: {}", selfId, attempt);
//判断销毁状态
if (!destroyInProgress.get()) {
//初始化 Actor
actor.init(this);
if (!destroyInProgress.get()) {
//设置状态
ready.set(READY);
//尝试处理队列消息
tryProcessQueue(false);
}
}
} catch (Throwable t) {
log.debug("[{}] Failed to init actor, attempt: {}", selfId, attempt, t);
//尝试计数加一
int attemptIdx = attempt + 1;
//获取初始化失败策略
InitFailureStrategy strategy = actor.onInitFailure(attempt, t);
if (strategy.isStop() || (settings.getMaxActorInitAttempts() > 0 && attemptIdx > settings.getMaxActorInitAttempts())) {
//失败策略为停止或尝试次数已超过最大次数
log.info("[{}] Failed to init actor, attempt {}, going to stop attempts.", selfId, attempt, t);
//记录停止原因为初始化失败
stopReason = TbActorStopReason.INIT_FAILED;
//销毁
destroy();
} else if (strategy.getRetryDelay() > 0) {
log.info("[{}] Failed to init actor, attempt {}, going to retry in attempts in {}ms", selfId, attempt, strategy.getRetryDelay());
log.debug("[{}] Error", selfId, t);
//设置给定的延时后重新尝试初始化
system.getScheduler().schedule(() -> dispatcher.getExecutor().execute(() -> tryInit(attemptIdx)), strategy.getRetryDelay(), TimeUnit.MILLISECONDS);
} else {
log.info("[{}] Failed to init actor, attempt {}, going to retry immediately", selfId, attempt);
log.debug("[{}] Error", selfId, t);
//立即重新尝试初始化
dispatcher.getExecutor().execute(() -> tryInit(attemptIdx));
}
}
}
/**
* 消息入队,关键方法
*/
private void enqueue(TbActorMsg msg, boolean highPriority) {
//判断销毁状态
if (!destroyInProgress.get()) {
if (highPriority) {
//将消息放入高优先级队列
highPriorityMsgs.add(msg);
} else {
//将消息放入普通优先级队列
normalPriorityMsgs.add(msg);
}
//尝试处理队列消息
tryProcessQueue(true);
} else {
//处于销毁状态
if (highPriority && msg.getMsgType().equals(MsgType.RULE_NODE_UPDATED_MSG)) {
//当前为高优先级的规则节点更新消息
//加锁
synchronized (this) {
if (stopReason == TbActorStopReason.INIT_FAILED) {
//当前停止原因为初始化失败
//更改销毁状态
destroyInProgress.set(false);
//重置停止原因
stopReason = null;
//再次初始化 Actor
initActor();
} else {
//回调 Actor停止原因
msg.onTbActorStopped(stopReason);
}
}
} else {
//回调 Actor停止原因
msg.onTbActorStopped(stopReason);
}
}
}
private void tryProcessQueue(boolean newMsg) {
if (ready.get() == READY) {
//已初始化完成
if (newMsg || !highPriorityMsgs.isEmpty() || !normalPriorityMsgs.isEmpty()) {
//当前有新的消息或消息队列不为空(有待处理的消息)
//判断当前状态是否为空闲,并改为繁忙
if (busy.compareAndSet(FREE, BUSY)) {
//异步处理邮箱
dispatcher.getExecutor().execute(this::processMailbox);
} else {
//当前状态为繁忙
log.trace("[{}] MessageBox is busy, new msg: {}", selfId, newMsg);
}
} else {
log.trace("[{}] MessageBox is empty, new msg: {}", selfId, newMsg);
}
} else {
log.trace("[{}] MessageBox is not ready, new msg: {}", selfId, newMsg);
}
}
private void processMailbox() {
//标记是否有更多的消息
boolean noMoreElements = false;
//根据指定的吞吐量遍历处理
for (int i = 0; i < settings.getActorThroughput(); i++) {
//从高优先级队列获取消息
TbActorMsg msg = highPriorityMsgs.poll();
if (msg == null) {
//从普通优先级队列获取消息
msg = normalPriorityMsgs.poll();
}
if (msg != null) {
try {
log.debug("[{}] Going to process message: {}", selfId, msg);
//调用 Actor 处理消息
actor.process(msg);
} catch (TbRuleNodeUpdateException updateException) {
//规则节点更新异常视为初始化失败
stopReason = TbActorStopReason.INIT_FAILED;
//销毁
destroy();
} catch (Throwable t) {
log.debug("[{}] Failed to process message: {}", selfId, msg, t);
//获取处理失败策略
ProcessFailureStrategy strategy = actor.onProcessFailure(t);
if (strategy.isStop()) {
//停止 Actor
system.stop(selfId);
}
}
} else {
//没有未处理的消息
noMoreElements = true;
break;
}
}
if (noMoreElements) {
//设置空闲状态
busy.set(FREE);
//尝试处理队列消息(再次检查)
dispatcher.getExecutor().execute(() -> tryProcessQueue(false));
} else {
//继续处理邮箱
dispatcher.getExecutor().execute(this::processMailbox);
}
}
@Override
public TbActorId getSelf() {
return selfId;
}
@Override
public void tell(TbActorId target, TbActorMsg actorMsg) {
system.tell(target, actorMsg);
}
@Override
public void broadcastToChildren(TbActorMsg msg) {
system.broadcastToChildren(selfId, msg);
}
@Override
public void broadcastToChildrenByType(TbActorMsg msg, EntityType entityType) {
broadcastToChildren(msg, actorId -> entityType.equals(actorId.getEntityType()));
}
@Override
public void broadcastToChildren(TbActorMsg msg, Predicate<TbActorId> childFilter) {
system.broadcastToChildren(selfId, childFilter, msg);
}
@Override
public List<TbActorId> filterChildren(Predicate<TbActorId> childFilter) {
return system.filterChildren(selfId, childFilter);
}
@Override
public void stop(TbActorId target) {
system.stop(target);
}
@Override
public TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier<String> dispatcher, Supplier<TbActorCreator> creator) {
TbActorRef actorRef = system.getActor(actorId);
if (actorRef == null) {
return system.createChildActor(dispatcher.get(), creator.get(), selfId);
} else {
return actorRef;
}
}
public void destroy() {
if (stopReason == null) {
stopReason = TbActorStopReason.STOPPED;
}
destroyInProgress.set(true);
dispatcher.getExecutor().execute(() -> {
try {
ready.set(NOT_READY);
actor.destroy();
highPriorityMsgs.forEach(msg -> msg.onTbActorStopped(stopReason));
normalPriorityMsgs.forEach(msg -> msg.onTbActorStopped(stopReason));
} catch (Throwable t) {
log.warn("[{}] Failed to destroy actor: {}", selfId, t);
}
});
}
@Override
public TbActorId getActorId() {
return selfId;
}
@Override
public void tell(TbActorMsg actorMsg) {
enqueue(actorMsg, NORMAL_PRIORITY);
}
@Override
public void tellWithHighPriority(TbActorMsg actorMsg) {
enqueue(actorMsg, HIGH_PRIORITY);
}
}
综上,Actor
的大致逻辑如下:
- 创建DefaultTbActorSystem作为入口
- 通过DefaultTbActorSystem创建TbActorMailbox(包括当前DefaultTbActorSystem,
Actor
实例,父Actor
引用,调度器等) - TbActorMailbox根据需求
- 调用
Actor
处理消息 - 通过DefaultTbActorSystem创建新的TbActorMailbox并发送消息
- 调用
所有TbActorMailbox由DefaultTbActorSystem统一管理
定义于common/message模块 ↩︎