C#使用“分块处理”和“并行处理”优化代码

模拟处理10万条List集合中的数据，

List<PeoMterModel> peoMterModels = new List<PeoMterModel>();
for (int i = 0; i < 100000; i++) 
{
    PeoMterModel peoMterModel=new PeoMterModel();
    peoMterModel.Id=Guid.Empty;
    peoMterModel.Name= i.ToString();
    peoMterModels.Add(peoMterModel);
}

设置每次处理20000条，判断需要多少个线程，用线程并行处理处理数据。每次处理2000条，则需要5个线程进行处理，计算需要设置多少个线程

int batchSize = 20000;
int numberOfTasks = (int)Math.Ceiling((double)peoMterModels.Count / batchSize);

用Task集合进行处理这些数据

var tasks = new List<Task>();
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
for (int i = 0; i < numberOfTasks; i++)
{
    int start = i * batchSize;
    int end = Math.Min(start + batchSize, peoMterModels.Count);
    var batch = peoMterModels.GetRange(start, end - start);

    // 创建并启动任务
    var task = Task.Run(() => ProcessBatch(batch));
    tasks.Add(task);
}

// 等待所有任务完成
Task.WhenAll(tasks).Wait();
stopwatch.Stop();
TimeSpan elapsed = stopwatch.Elapsed;
Console.WriteLine($"Method execution time: {elapsed.TotalMilliseconds} ms");

void ProcessBatch(List<PeoMterModel> peoMterModels)
{
    for (int i = 0; i < peoMterModels.Count; i++) 
    {
        peoMterModels[i].Id = GetNew();
    }
}

Guid GetNew() 
{
    Guid guid = Guid.NewGuid();
    string aa = new Random().Next(100,20000).ToString();
    string bb = new Random().Next(156, 73751).ToString();
    string cc = new Random().Next(442, 58111).ToString();
    string dd = new Random().Next(3698, 14521).ToString();
    string ee = new Random().Next(1525, 69531).ToString();
    string f = new Random().Next(1455, 85231).ToString();
    double gjk= Convert.ToDouble((Convert.ToInt32(aa).ToString()))+ Convert.ToDouble((Convert.ToInt32(aa).ToString())) + Convert.ToDouble((Convert.ToInt32(bb).ToString())) + 
        Convert.ToDouble((Convert.ToInt32(cc).ToString())) + Convert.ToDouble((Convert.ToInt32(dd).ToString())) + Convert.ToDouble((Convert.ToInt32(ee).ToString()));
    double dadas = Convert.ToDouble((Convert.ToInt32(f).ToString())) / Convert.ToDouble((Convert.ToInt32(cc).ToString())) +
        Convert.ToDouble((Convert.ToInt32(aa).ToString())) + Convert.ToDouble((Convert.ToInt32(aa).ToString())) + Convert.ToDouble((Convert.ToInt32(bb).ToString())) +
        Convert.ToDouble((Convert.ToInt32(cc).ToString())) + Convert.ToDouble((Convert.ToInt32(dd).ToString())) + Convert.ToDouble((Convert.ToInt32(ee).ToString()))
        + Convert.ToDouble((Convert.ToInt32(gjk.ToString()).ToString()));

    string aa1 = new Random().Next(55, 75232).ToString();
    string bb1 = new Random().Next(4455, 18523).ToString();
    string cc1 = new Random().Next(155, 85244).ToString();
    string dd1 = new Random().Next(1875, 91523).ToString();
    string ee1 = new Random().Next(1455, 18523).ToString();
    string f1 = new Random().Next(1, 86001).ToString();

    double gjk2 = Convert.ToDouble((Convert.ToInt32(aa1).ToString())) + Convert.ToDouble((Convert.ToInt32(bb1).ToString())) + Convert.ToDouble((Convert.ToInt32(cc1).ToString())) +
        Convert.ToDouble((Convert.ToInt32(dd1).ToString())) + Convert.ToDouble((Convert.ToInt32(ee1).ToString())) + Convert.ToDouble((Convert.ToInt32(f1).ToString()));
    string sdad111 = (Convert.ToDouble((Convert.ToInt32(gjk).ToString())) + 
        Convert.ToDouble((Convert.ToInt32(dadas).ToString())) + Convert.ToDouble((Convert.ToInt32(gjk2).ToString()))).ToString();
    string sdad222 = "qwerwtqwtqfwqtywfqhwghqgwhwg"+$"{Convert.ToDouble((Convert.ToInt32(dadas).ToString()))+ Convert.ToDouble((Convert.ToInt32(gjk2).ToString()))}"+"fasdfsaf";

    PeoMterModel peoMterModel = new PeoMterModel();
    peoMterModel.GetModels();

    string json = JsonConvert.SerializeObject(peoMterModel);

    //Thread.Sleep(1000);
    //Thread.Sleep(100);
    //GC.Collect();
    return guid;
}

以上代码可以看到GetNew()方法有很多计算，还有序列化，模仿生产环境中的耗时操作；

执行完成后，发现执行时间为5276.2017ms

接下来直接测试，使用循环的方式进行处理，直接循环这10万条数据，依旧调用GetNew();这个耗时方式

Stopwatch stopwatch1 = new Stopwatch();
stopwatch1.Start();
//普通执行方法
for (int i = 0; i < peoMterModels.Count; i++)
{
    peoMterModels[i].Id = GetNew();
}
stopwatch1.Stop();
TimeSpan elapsed1 = stopwatch1.Elapsed;
Console.WriteLine($"Method execution time: {elapsed1.TotalMilliseconds} ms");

测试后发现需要15076.1498ms，相对于使用并行，速度增加了3倍

综上所述，建议使用“分块处理”和“并行处理”。