执行任务
通过fx.invoke我们可以找到任务的执行入口:
代码路径:temporal/service/history/replication/fx.go
通过调用关系,我们找到关键的入口方法如下:
func (s *SequentialScheduler[T]) Start() {
if !atomic.CompareAndSwapInt32(
&s.status,
common.DaemonStatusInitialized,
common.DaemonStatusStarted,
) {
return
}
s.startWorkers(s.options.WorkerCount())
s.shutdownWG.Add(1)
go s.workerMonitor()
s.logger.Info("sequential scheduler started")
}
上面的逻辑就是:
启动执行任务的worker
通过一个协程来监控worker:新增或减少
下面来具体看下启动woker的代码:
func (s *SequentialScheduler[T]) startWorkers(
count int,
) {
for i := 0; i < count; i++ {
shutdownCh := make(chan struct{})
s.workerShutdownCh = append(s.workerShutdownCh, shutdownCh)
s.shutdownWG.Add(1)
go s.pollTaskQueue(shutdownCh)
}
}
上面代码大致如下:
根据配置来启动指定数量的worker
每个worker是一个协程:拉取任务队列中的任务并执行
下面来看看拉取任务的代码:
func (s *SequentialScheduler[T]) pollTaskQueue(workerShutdownCh <-chan struct{}) {
defer s.shutdownWG.Done()
for {
select {
case <-s.shutdownChan:
s.drainTasks()
return
case <-workerShutdownCh:
return
case queue := <-s.queueChan:
s.processTaskQueue(queue, workerShutdownCh)
}
}
}
上面代码大致如下:
启动一个loop循环
从队列chan中获取任务队列
处理该任务队列
下面来看看处理任务的逻辑:
func (s *SequentialScheduler[T]) processTaskQueue(
queue SequentialTaskQueue[T],
workerShutdownCh <-chan struct{},
) {
for {
select {
case <-s.shutdownChan:
s.drainTasks()
return
case <-workerShutdownCh:
// Put queue back to the queue channel
s.queueChan <- queue
return
default:
if !queue.IsEmpty() {
s.executeTask(queue)
} else {
deleted := s.queues.RemoveIf(queue.ID(), func(key interface{}, value interface{}) bool {
return value.(SequentialTaskQueue[T]).IsEmpty()
})
if deleted {
return
}
// if deletion failed, meaning that task queue is offered with new task
// continue execution
}
}
}
}
func (s *SequentialScheduler[T]) executeTask(queue SequentialTaskQueue[T]) {
task := queue.Remove()
operation := func() error {
if err := task.Execute(); err != nil {
return task.HandleErr(err)
}
return nil
}
isRetryable := func(err error) bool {
return !s.isStopped() && task.IsRetryableError(err)
}
if err := backoff.ThrottleRetry(operation, task.RetryPolicy(), isRetryable); err != nil {
if s.isStopped() {
task.Abort()
return
}
task.Nack(err)
return
}
task.Ack()
}
上面代码逻辑如下:
启动一个loop循环
从任务队列中不断的获取任务
然后执行该任务
发送任务
看到这里我们有个疑问了,这些任务队列是怎么放入chan里面的呢?下面来看看
同样的,在fx的invoke方法,调用了事件流逻辑:
StreamWorkflowReplicationMessages
我们通过层层调用可以找到这样的代码:
func (m *StreamReceiverMonitorImpl) eventLoop() {
defer m.Stop()
ticker := time.NewTicker(streamReceiverMonitorInterval)
defer ticker.Stop()
clusterMetadataChangeChan := make(chan struct{}, 1)
m.ClusterMetadata.RegisterMetadataChangeCallback(m, func(_ map[string]*cluster.ClusterInformation, _ map[string]*cluster.ClusterInformation) {
select {
case clusterMetadataChangeChan <- struct{}{}:
default:
}
})
defer m.ClusterMetadata.UnRegisterMetadataChangeCallback(m)
m.reconcileOutboundStreams()
Loop:
for !m.shutdownOnce.IsShutdown() {
select {
case <-clusterMetadataChangeChan:
m.reconcileInboundStreams()
m.reconcileOutboundStreams()
case <-ticker.C:
m.reconcileInboundStreams()
m.reconcileOutboundStreams()
case <-m.shutdownOnce.Channel():
break Loop
}
}
}
func (m *StreamReceiverMonitorImpl) doReconcileOutboundStreams(
streamKeys map[ClusterShardKeyPair]struct{},
) {
m.Lock()
defer m.Unlock()
if m.shutdownOnce.IsShutdown() {
return
}
for streamKey, stream := range m.outboundStreams {
if !stream.IsValid() {
stream.Stop()
delete(m.outboundStreams, streamKey)
} else if _, ok := streamKeys[streamKey]; !ok {
stream.Stop()
delete(m.outboundStreams, streamKey)
}
}
for streamKey := range streamKeys {
if _, ok := m.outboundStreams[streamKey]; !ok {
stream := NewStreamReceiver(
m.ProcessToolBox,
streamKey.Client,
streamKey.Server,
)
stream.Start()
m.outboundStreams[streamKey] = stream
}
}
}
从这里可以看到,调用了stramReceiverImpl的start方法,我们找到这个方法:
func (r *StreamReceiverImpl) Start() {
if !atomic.CompareAndSwapInt32(
&r.status,
common.DaemonStatusInitialized,
common.DaemonStatusStarted,
) {
return
}
go r.sendEventLoop()
go r.recvEventLoop()
r.logger.Info("StreamReceiver started.")
}
上面的逻辑大致如下:
启动协程:发送事件
启动协程:接收事件
下面来看看接收事件的代码:
func (r *StreamReceiverImpl) processMessages(
stream Stream,
) error {
allClusterInfo := r.ClusterMetadata.GetAllClusterInfo()
clusterName, _, err := ClusterIDToClusterNameShardCount(allClusterInfo, r.serverShardKey.ClusterID)
if err != nil {
return err
}
streamRespChen, err := stream.Recv()
if err != nil {
r.logger.Error("StreamReceiver unable to recv message, err", tag.Error(err))
return err
}
for streamResp := range streamRespChen {
if streamResp.Err != nil {
r.logger.Error("StreamReceiver recv stream encountered unexpected err", tag.Error(streamResp.Err))
return streamResp.Err
}
tasks := r.ConvertTasks(
clusterName,
r.clientShardKey,
r.serverShardKey,
streamResp.Resp.GetMessages().ReplicationTasks...,
)
exclusiveHighWatermark := streamResp.Resp.GetMessages().ExclusiveHighWatermark
exclusiveHighWatermarkTime := timestamp.TimeValue(streamResp.Resp.GetMessages().ExclusiveHighWatermarkTime)
for _, task := range r.taskTracker.TrackTasks(WatermarkInfo{
Watermark: exclusiveHighWatermark,
Timestamp: exclusiveHighWatermarkTime,
}, tasks...) {
r.ProcessToolBox.TaskScheduler.Submit(task)
}
}
r.logger.Error("StreamReceiver encountered channel close")
return nil
}
上面代码逻辑如下:
从事件流中获取消息
将消息转换为task对象
将task对象提交给任务调度器
那么事件流中的任务时怎么来的呢?这里就不展开的,其实就是通过history的rpc接口,从数据库中读取出来,发送到事件流中的。
这里的事件机制运用了发布订阅模式。