C# 多线程+队列处理大批量数据，进而缩短处理时间

转载连接： https://blog.csdn.net/wangtinglong1/article/details/78890645

public void DealData(){

                int pageSize = 200;

//创建队列

                        var queue = new MessageQueueManager<Model>();

                        queue.Setup();
                        //开启生产者线程
                        var producerManager = new TaskManager();
                        producerManager.Setup(5);
                        producerManager.Start((index) =>
                        {
                            var pageIndex = Convert.ToInt32(index) + 1;
                            while (true)
                            {
                                try
                                {
                                    var modelList = GetModelList(pageIndex,pageSize);

                                    if (modelList == null)
                                    {
                                        break;
                                    }

                                    //循环学习统计加入消费者队列
                                    foreach (var model in modelList )
                                    {
                                        //自旋
                                        while (!queue.IsCanEnqueue)
                                        {
                                            Thread.Sleep(5 * 1000);
                                        }
                                        queue.Enqueue(model);
                                    }
                                    if (modelList.Count < pageSize)
                                    {
                                        break;
                                    }
                                    pageIndex += taskCount;
                                }
                                catch (Exception ex)
                                {
                                    pageIndex += taskCount;
                                    if (pageIndex > 2000)
                                    {
                                        break;
                                    }
                                    continue;
                                }
                            }
                        });

                        //消费者线程
                        var consumerManager = new TaskManager();
                        //这里设置一个线程进行处理数据
                        consumerManager.Setup(1);
                        consumerManager.Start(() =>
                        {
                            //从队列中取出数据进行处理
                            queue.Dequeue((info) =>
                            {
                                try
                                {
   //处理数据

                                   // AddOrUpdate( info);
                                }
                                catch (Exception ex)
                                {
                                    
                                }
                            });
                        });

                        producerManager.Wait();
                        queue.Cancel();
                        consumerManager.Wait();

}

/// <summary>
    /// 消息器
    /// </summary>
    public class MessageQueueManager<T>
    {
        private readonly BlockingCollection<T> _messageCollection = new BlockingCollection<T>(10000);

        private CancellationTokenSource _cancellationTokenSource;
        private CancellationToken _cancellationToken;

        public void Enqueue(T message)
        {
            if (_messageCollection.Count < _messageCollection.BoundedCapacity)
                _messageCollection.Add(message);
            if (_cancellationToken.IsCancellationRequested && !_messageCollection.IsCompleted)
                _messageCollection.CompleteAdding();
        }

        public void Dequeue(Action<T> writeAction)
        {
            foreach (var message in _messageCollection.GetConsumingEnumerable())
            {
                writeAction(message);
                Thread.Sleep(10);
            }
        }

        public void Setup()
        {
            _cancellationTokenSource = new CancellationTokenSource();
            _cancellationToken = _cancellationTokenSource.Token;
        }

        public void Cancel()
        {
            if (!_cancellationTokenSource.IsCancellationRequested)
                _cancellationTokenSource.Cancel(false);
            if (!_messageCollection.IsCompleted) _messageCollection.CompleteAdding();
        }

        public bool IsCanEnqueue
        {
            get
            {
                return _messageCollection.BoundedCapacity > _messageCollection.Count;
            }
        }
    }

//线程管理器

 public class TaskManager
    {
        public List<Task> _taskList;
        public int _total;

        public void Setup(int count)
        {
            _total = count;
            _taskList = new List<Task>(count);
        }

        public void Start(Action doing)
        {
            for (int i = 0; i < _total; i++)
            {
                var task = Task.Factory.StartNew(doing);
                _taskList.Add(task);
            }
        }

        public void Start(Action<int> doing)
        {
            for (int i = 0; i < _total; i++)
            {
                var task = Task.Factory.StartNew((index) => doing((int)index), i);
                _taskList.Add(task);
            }
        }

        public void Wait()
        {
            Task.WaitAll(_taskList.ToArray());
        }
    }
--------------------- 
作者：wangtinglong1 
来源：CSDN 
原文：https://blog.csdn.net/wangtinglong1/article/details/78890645 
版权声明：本文为博主原创文章，转载请附上博文链接！