C# 多条流水线异步工作

首先放上泛型封装的代码：

using System;
using System.Collections;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;

namespace TEST
{
public abstract class IPipelineStep<DATA> where DATA: DATAModel
{  
    private static readonly object obj = new object();
    public int step;
    public bool IsFirstRun = true;
    public delegate void CompleteMessageHander(DATA input);
    public event CompleteMessageHander completeEvent;        
    protected abstract bool Process(DATA input);
    private void CallBack(DATA input)
    {
        bool result = false;
        try
        {               
             result = Process(input);
        }
        catch(Exception ex)
        {
            LogExt.AddSysLog(ex, GetType().Name);               
        }
        input.step += 5;
        if (result)
        {
            completeEvent?.Invoke(input);
        }
        else
        {
            input.hasComplete = true;
            input.AllPassed = false;
            input.step = -2;//表示中途失败
            completeEvent?.Invoke(input);
        }
    }
    public void Run(DATA input)
    {           
        IsFirstRun = false;
        lock (obj)
        {
            if (input.step > step * 10 - 5|| input.hasComplete)
            {
                completeEvent?.Invoke(null);
                return;
            }
            input.step += 5;               
        }            
        Task.Run(() => CallBack(input));
    }
}
public class PipelineStepExtensions<DATA> where DATA : DATAModel
{
    public delegate void PipelineStepHander(object data, PipelineStepExtensions<DATA> owner);
    private static readonly object obj = new object();
    public event PipelineStepHander OnePieceCompleted;
    public event PipelineStepHander AnyStepChange;
    private List<DATA> oglist;
    public event PipelineStepHander AllCompleted;        
    private List<Type> _steps;//类型基于IPipelineStep<DATA>
    private Dictionary<int, ConcurrentQueueHelper<DATA>> DATABuffer;//蓄水池
    private Dictionary<int, List<IPipelineStep<DATA>>> listthread;
    public PipelineStepExtensions(List<DATA> list) : this(list, null)
    {           
    }        
    /// <summary>
    /// 构造函数
    /// </summary>
    /// <param name="list">原始数据</param>
    /// <param name="steps">传入的类型必须基于IPipelineStep</param>
    public PipelineStepExtensions(List<DATA> orgdata, List<Type> steps)
    {
        if (Config.记录函数时间)
            LogExt.AddInfor(Config.时间性能记录路径, "导入开始时间:" + DateTime.Now.ToString());
        _steps = steps;
        if (_steps == null)
            _steps = new List<Type>();
        DATABuffer = new Dictionary<int, ConcurrentQueueHelper<DATA>>();
        oglist = orgdata;
        oglist.ForEach(x => 
        {
            x.StepChangeEvent -= X_StepChangeEvent;
            x.HasCompleteEvent -= X_HasCompleteEvent;
            x.step = 5;                
            x.hasComplete = false;
            x.AllPassed = false;
            x.HasCompleteEvent += X_HasCompleteEvent;              
            x.StepChangeEvent += X_StepChangeEvent; 
        });
        DATABuffer[0] = new ConcurrentQueueHelper<DATA>(orgdata);
        DATABuffer[0].EnqueueEvent += (obj, y) => DATABuffer_EnqueueEvent(DATABuffer[0], listthread[1]);
        listthread = new Dictionary<int, List<IPipelineStep<DATA>>>();
    }

    private void X_StepChangeEvent(object sender, EventArgs e)
    {
        AnyStepChange?.Invoke(sender, this);
    }
    public void Release()
    {
        oglist.ForEach(x => 
        {
            x.HasCompleteEvent -= X_HasCompleteEvent;
            x.StepChangeEvent -= X_StepChangeEvent;
        });
    }
    private void X_HasCompleteEvent(object sender, EventArgs e)
    {
        OnePieceCompleted?.Invoke(sender, this);
    }

    public decimal GetProgress()
    {
        var 步骤 = _steps.Count;
        int 进度 = oglist.Where(x => x.step < 0).Count() * 步骤 + oglist.Sum(x => (x.step - 5) / 10);
        int 总进度 = oglist.Count * 步骤;
        return (decimal)进度 / (decimal)总进度;
    }
    public int GetPipeBsuyCount()
    {
        return oglist.Where(y => y.step % 10 == 0).Count();
    }
    public PipelineStepExtensions<DATA> AddStep(Type step)
    {
        _steps.Add(step);
        return this;
    }
    private void DATABuffer_EnqueueEvent(ConcurrentQueueHelper<DATA> cq, List<IPipelineStep<DATA>> list)
    {
        lock (obj)
        {
            if (cq.Count == 0)
                return;
            //入队消息得到后，查询是否有空闲的下一轮处理模块，若有则提取并逐一输入
            var qq = list.Where(x => x.IsFirstRun);
            foreach (var it in qq)
            {
                if (!it.IsFirstRun)
                    continue;
                bool bl = cq.TryDequeue(out DATA dt);
                if (bl)
                    it.Run(dt);
            }
        }     
    }
    public void Begin(int ThreadNum)
    {
        if (oglist.Count == 0)
            throw new Exception("原始数据不能为空");
        if (_steps.Count == 0)
            throw new Exception("步骤不能为空");
        Init(ThreadNum);
        DATABuffer_EnqueueEvent(DATABuffer[0], listthread[1]);
    }      
    private void Init(int ThreadNum)
    {
        if (ThreadNum < 1)
            ThreadNum = 1;
        else
            ThreadNum = Math.Min(ThreadNum, oglist.Count);
        for (int i = 1; i <= _steps.Count; ++i)
        {                
            Type step = _steps[i - 1];
            listthread[i] = new List<IPipelineStep<DATA>>();
            if (i < _steps.Count)
            {
                DATABuffer[i] = new ConcurrentQueueHelper<DATA>();
                DATABuffer[i].step = i;
                DATABuffer[i].EnqueueEvent += (eq, data) => DATABuffer_EnqueueEvent(eq, listthread[eq.step + 1]);
            }
            //建立多条流水线
            for (var j = 0; j < ThreadNum; ++j)
            {
                var item = Activator.CreateInstance(step) as IPipelineStep<DATA>;
                if (item == null)
                    throw new Exception("Type类型必须继承自IPipelineStep<DATA>");
                item.step = i;
                item.completeEvent += data =>
                {
                    if (data != null && data.step < 0)//失败就是-2
                    {
                        if (oglist.FirstOrDefault(x => !x.hasComplete) == null)
                        {
                            AllCompleted?.Invoke(item, this);
                            return;
                        }
                    }
                    if (item.step == _steps.Count)
                    {
                        if (data != null)
                        {
                            data.AllPassed = true;
                            data.step = -1;//-1表示成功执行全部
                            data.hasComplete = true;
                        }                           
                        if (oglist.FirstOrDefault(x => !x.hasComplete) == null)
                        {
                            AllCompleted?.Invoke(item, this);
                            return;
                        }
                    }
                    //从上一级蓄水池预备获取
                    bool bl = DATABuffer[item.step - 1].TryDequeue(out DATA dt);
                    if (bl)
                        item.Run(dt);
                    else
                        item.IsFirstRun = true;//在无数据的时候转为首次，这样在数据新增时可以被检测并继续
                    if (item.step < _steps.Count)
                    {
                        //完成的部分进入蓄水池预备传给下一级
                        DATABuffer[item.step].Enqueue(data);
                    }
                    //item.IsFirstRun = true;
                    //DATABuffer_EnqueueEvent(DATABuffer[item.step - 1], listthread[item.step]);                     
                };
                listthread[i].Add(item);
            }                
        }         
    }
}
public class ConcurrentQueueHelper<T>
{
    public int Count => _queue.Count;
    public int step;
    public delegate void ConcurrentQueueHandel(ConcurrentQueueHelper<T> my, object data);
    public event ConcurrentQueueHandel EnqueueEvent;
    public event ConcurrentQueueHandel DequeueEvent;
    private ConcurrentQueue<T> _queue;
    public ConcurrentQueueHelper()
    {
        _queue = new ConcurrentQueue<T>();
    }
    public ConcurrentQueueHelper(List<T> list)
    {
        _queue = new ConcurrentQueue<T>(list);
    }       
    public void Enqueue(T item)
    {
        if (item != null)
            _queue.Enqueue(item);
        EnqueueEvent?.Invoke(this, item);
    }
    public void EnqueueNotEvent(T item)
    {
        if (item == null)
            return;
        _queue.Enqueue(item);
    }
    public bool TryDequeue(out T item)
    {
        bool bl;
        do
        {
            bl = _queue.TryDequeue(out item);
            if (!bl && _queue.Count == 0)
                return false;
        } while (!bl);
        DequeueEvent?.Invoke(this, item);
        return bl;
    }
    public bool TryDequeueNotEvent(out T item)
    {
        bool bl;
        do
        {
            bl = _queue.TryDequeue(out item);
            if (!bl && _queue.Count == 0)
                return false;
        } while (!bl);
        return bl;
    }
}
public class DATAModel
{
    public event EventHandler HasCompleteEvent;
    public event EventHandler StepChangeEvent;
    private bool _complete { get; set; }
    /// <summary>
    /// 用于统计完成数
    /// </summary>
    public bool hasComplete 
    {
        get => _complete; 
        set 
        {
            if (_complete == value)
                return;
            _complete = value;
            HasCompleteEvent?.Invoke(this, null);
        } 
    }
    private int _step = 5;
    /// <summary>
    /// 当前第几步,数值放大十倍，10表示第一步，15表示第一步结束，还未进入第二步，20表示第二步，-1为全部完成，-2为中途失败
    /// </summary>
    public int step
    {
        get => _step;
        set 
        {
            if (_step == value)
                return;
            _step = value;
            StepChangeEvent?.Invoke(this, null);
        }
    }
    public bool AllPassed { get; set; } = false;
}
}

下面是调用示例

using System;
using System.Collections.Generic;
using System.Threading;

namespace TEST
{
    public class 流水线测试程序
    {
        public void Test()
        {
            ManualResetEvent signal = new ManualResetEvent(false);
            var list = new List<流水线测试model>();
            for (var i = 0; i < 1000; ++i)
            {
                list.Add(new 流水线测试model());
            }
            var ll = new PipelineStepExtensions<流水线测试model>(list, new List<Type> { typeof(流程1), typeof(流程2), typeof(流程3), typeof(流程4) });
            //构造数据及步骤还可以写为如下
            //var ll = new PipelineStepExtensions<FileExplorerItemGroup>(list);
            //ll.AddStep(typeof(流程1)).AddStep(typeof(流程2)).AddStep(typeof(流程3)).AddStep(typeof(流程4)));      
            ll.AllCompleted += (x, y) =>
            {
                Console.WriteLine("OK");
                signal.Set();
                signal.Dispose();
            };
            ll.Begin(3);//同时开3条流水线，每条流水线的4个流程可互通，某一流程耗时，可使数据走同一层流程的其他流水线
            signal.WaitOne();
            ll.Release();
        }
    }
    public class 流水线测试model : DATAModel
    {
        public string Text { get; set; }
    }
    public class 流程1 : IPipelineStep<流水线测试model>
    {
        protected override bool Process(流水线测试model input)
        {
            input.Text = "第1步";
            Thread.Sleep(100);
            Console.WriteLine(input.Text + input.step);            
            return true;//false会被当做任务失败抛弃
        }
    }
    public class 流程2 : IPipelineStep<流水线测试model>
    {
        protected override bool Process(流水线测试model input)
        {
            input.Text = "第2步";
            Thread.Sleep(100);
            Console.WriteLine(input.Text + input.step);
            return true;//false会被当做任务失败抛弃
        }
    }
    public class 流程3 : IPipelineStep<流水线测试model>
    {
        protected override bool Process(流水线测试model input)
        {
            input.Text = "第3步";
            Thread.Sleep(100);
            Console.WriteLine(input.Text + input.step);
            return true;//false会被当做任务失败抛弃
        }
    }
    public class 流程4 : IPipelineStep<流水线测试model>
    {
        protected override bool Process(流水线测试model input)
        {
            input.Text = "第4步";
            Thread.Sleep(100);
            Console.WriteLine(input.Text + input.step);           
            return true;//false会被当做任务失败抛弃
        }
    }
}