c#：实现动态编译，并实现动态MultiProcess功能（来自python multiprocess的想法）

最新推荐文章于 2022-01-10 10:33:59 发布

weixin_33725272

最新推荐文章于 2022-01-10 10:33:59 发布

阅读量113

点赞数

文章标签： python c# 数据库

　　由于之前一直遇到一些关于并行进行数据处理的时效果往往不好，不管是c#还是java程序都是一样，但是在Python中通过multiprocess实现同样的功能时，却发现确实可以提高程序运行的性能，及服务器资源使用提高。python具体性能及multiprocess用法，请参考：《Python：使用pymssql批量插入csv文件到数据库测试》

　　如有转载请标明原文地址：http://www.cnblogs.com/yy3b2007com/p/7228337.html

　　很久之前就设想如何在c#中实现多进程的方案，如今终于设计出了C#中动态实现多进程的方案：

具体代码：

  1 using Microsoft.CSharp;
  2 using System;
  3 using System.CodeDom;
  4 using System.CodeDom.Compiler;
  5 using System.Collections.Generic;
  6 using System.Linq;
  7 using System.Reflection;
  8 using System.Text;
  9 
 10 namespace MutilProcessDemo
 11 {
 12     public class Program
 13     {
 14         static System.Timers.Timer timer = null;
 15 
 16         public static void Main(string[] args)
 17         {
 18             GenerateExe();
 19 
 20             timer = new System.Timers.Timer();
 21             timer.AutoReset = true;
 22             timer.Enabled = true;
 23             timer.Interval = 5000;
 24             timer.Elapsed += Timer_Elapsed;
 25 
 26 
 27             Console.WriteLine("Press Enter key to exit...");
 28             Console.ReadKey();
 29         }
 30 
 31         private static void DynamicCompiler()
 32         {
 33             // 1.CSharpCodePrivoder 
 34             CSharpCodeProvider objCSharpCodePrivoder = new CSharpCodeProvider();
 35 
 36             // 2.ICodeComplier 
 37             ICodeCompiler objICodeCompiler = objCSharpCodePrivoder.CreateCompiler();
 38 
 39             // 3.CompilerParameters 
 40             CompilerParameters objCompilerParameters = new CompilerParameters();
 41             objCompilerParameters.ReferencedAssemblies.Add("System.dll");
 42             objCompilerParameters.GenerateExecutable = false;
 43             objCompilerParameters.GenerateInMemory = true;
 44 
 45             // 4.CompilerResults 
 46             CompilerResults cr = objICodeCompiler.CompileAssemblyFromSource(objCompilerParameters, GenerateCode());
 47 
 48             if (cr.Errors.HasErrors)
 49             {
 50                 Console.WriteLine("编译错误：");
 51                 foreach (CompilerError err in cr.Errors)
 52                 {
 53                     Console.WriteLine(err.ErrorText);
 54                 }
 55             }
 56             else
 57             {
 58                 // 通过反射，调用HelloWorld的实例 
 59                 Assembly objAssembly = cr.CompiledAssembly;
 60                 object objHelloWorld = objAssembly.CreateInstance("DynamicCodeGenerate.HelloWorld");
 61                 MethodInfo objMI = objHelloWorld.GetType().GetMethod("OutPut");
 62 
 63                 Console.WriteLine(objMI.Invoke(objHelloWorld, null));
 64             }
 65 
 66             Console.ReadLine();
 67         }
 68 
 69         static string GenerateCode()
 70         {
 71             StringBuilder sb = new StringBuilder();
 72             sb.Append("using System;");
 73             sb.Append(Environment.NewLine);
 74             sb.Append("namespace DynamicCodeGenerate");
 75             sb.Append(Environment.NewLine);
 76             sb.Append("{");
 77             sb.Append(Environment.NewLine);
 78             sb.Append(" public class HelloWorld");
 79             sb.Append(Environment.NewLine);
 80             sb.Append(" {");
 81             sb.Append(Environment.NewLine);
 82             sb.Append(" public string OutPut()");
 83             sb.Append(Environment.NewLine);
 84             sb.Append(" {");
 85             sb.Append(Environment.NewLine);
 86             sb.Append(" return \"Hello world!\";");
 87             sb.Append(Environment.NewLine);
 88             sb.Append(" }");
 89             sb.Append(Environment.NewLine);
 90             sb.Append(" }");
 91             sb.Append(Environment.NewLine);
 92             sb.Append("}");
 93 
 94             string code = sb.ToString();
 95             Console.WriteLine(code);
 96             Console.WriteLine();
 97 
 98             return code;
 99         }
100 
101 
102         private static void Timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
103         {
104             System.Diagnostics.Process[] processes = System.Diagnostics.Process.GetProcessesByName("HelloWorld");
105 
106             int maxProcessCount = 8;
107             // 如果超过了最大允许任务数据不再触发新的进程任务
108             if (processes.Length >= maxProcessCount)
109                 return;
110 
111             List<int> tasks = new List<int>();
112             Random random = new Random(1000);
113             for (int i = 0; i < maxProcessCount - processes.Length; i++)
114             {
115                 tasks.Add(random.Next(1000));
116             }
117 
118 
119             System.Threading.Tasks.Task[] taskItems = new System.Threading.Tasks.Task[tasks.Count];
120 
121             for (int i = 0; i < tasks.Count; i++)
122             {
123                 //TaskBase taskSubmit = new ParseMrToHdfsFileItemTask();
124                 //taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew(taskSubmit.Submit, new ParseMrToHdfsFileItemTaskArgument() { Task = tasks[i], ComputeNode = this.ComputeNode }, TaskCreationOptions.PreferFairness);
125                 taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew((object state) =>
126                 {
127                     System.Diagnostics.Process process = new System.Diagnostics.Process();
128                 //System.Diagnostics.Process process = System.Diagnostics.Process.Start("HelloWorld.exe", state.ToString());
129                 process.StartInfo = new System.Diagnostics.ProcessStartInfo();
130                     process.StartInfo.CreateNoWindow = true;
131                     process.StartInfo.UseShellExecute = false;
132                     process.StartInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
133                     process.StartInfo.Arguments = state.ToString();
134                     process.StartInfo.FileName = "HelloWorld.exe";
135                     process.Start();
136                     process.WaitForExit();
137                     process.Close();
138                     process.Dispose();
139                 }, tasks[i]);
140             }
141 
142             System.Threading.Tasks.Task.WaitAll(taskItems);
143         }
144 
145         private static void GenerateExe()
146         {
147             // 创建编译器对象
148             CSharpCodeProvider p = new CSharpCodeProvider();
149             ICodeCompiler cc = p.CreateCompiler();
150 
151             // 设置编译参数
152             CompilerParameters options = new CompilerParameters();
153             options.ReferencedAssemblies.Add("System.dll");
154             options.ReferencedAssemblies.Add("MutilProcessDemo.exe");
155             options.GenerateExecutable = true;
156             options.OutputAssembly = "HelloWorld.exe";
157 
158             //options.ReferencedAssemblies.Add("System.Windows.Forms.dll");
159             //options.EmbeddedResources.Add("Data.xml"); // 添加内置资源
160             //options.CompilerOptions += " /target:winexe";
161             //options.CompilerOptions += " /res:Resource1.res";
162             //options.CompilerOptions += " /win32icon:test.ico";
163 
164             // 创建源码
165 
166             // 1. 使用CodeDom创建源码
167             //CodeCompileUnit cu = new CodeCompileUnit();
168             //CodeNamespace Samples = new CodeNamespace("Samples");
169             //cu.Namespaces.Add(Samples);
170             //Samples.Imports.Add(new CodeNamespaceImport("System"));
171             //CodeTypeDeclaration Class1 = new CodeTypeDeclaration("Class1");
172             //Samples.Types.Add(Class1);
173             //CodeEntryPointMethod Start = new CodeEntryPointMethod();
174             //CodeMethodInvokeExpression cs1 = new CodeMethodInvokeExpression(
175             // new CodeTypeReferenceExpression("System.Console"), "WriteLine",
176             // new CodePrimitiveExpression("Hello World!")
177             // );
178             //Start.Statements.Add(new CodeExpressionStatement(cs1));
179             //Class1.Members.Add(Start);
180 
181             // 2. 直接指定源码字符串
182             string code = @"
183 using System;
184 using MutilProcessDemo;
185 
186 namespace Samples
187 {
188     public class Class1
189     {
190         static void Main(string[] args)
191         {
192             Console.WriteLine(""Hello, World!"");
193             MutilProcessDemo.Program.DoMethod(args);
194             Console.WriteLine(DateTime.Now.ToString());
195         }
196     }
197 }";
198             CodeSnippetCompileUnit codeSnippetCompileUnit = new CodeSnippetCompileUnit(code);
199 
200             // 开始编译
201             CompilerResults compilerResults = cc.CompileAssemblyFromDom(options, codeSnippetCompileUnit);
202 
203             // 显示编译信息
204             if (compilerResults.Errors.Count == 0)
205                 Console.WriteLine("{0}compiled ok!", compilerResults.CompiledAssembly.Location);
206             else
207             {
208                 Console.WriteLine("Complie Error:");
209                 foreach (CompilerError error in compilerResults.Errors)
210                     Console.WriteLine(" {0}", error);
211             }
212         }
213 
214         public static void DoMethod(string[] args)
215         {
216             System.Console.WriteLine("begin ..." + args[0]);
217 
218             for (int i = 0; i < int.Parse(args[0]); i++)
219             {
220                 System.Threading.Thread.Sleep(20);
221             }
222 
223             System.Console.WriteLine("end..." + args[0]);
224         }
225     }
226 }

上边的程序运行之后会在\MutilProcessDemo\bin\Debug\下出现两个可执行的.exe：HelloWorld.exe、MutilProcessDemo.exe。

实战：

  1 public class Program_ForInsertATU : BaseDao
  2 {
  3     static readonly BuildingsBO buildBO = new BuildingsBO();
  4     static readonly FingerPrintDatabaseBO fingerPrintDatabaseBO = new FingerPrintDatabaseBO();
  5     static readonly GridMappingBuildingBO gridMappingBuildingBO = new GridMappingBuildingBO();
  6     static readonly SiteBO siteBO = new SiteBO();
  7     static NLog.Logger logger = new NLog.LogFactory().GetCurrentClassLogger();
  8 
  9     private static void GenerateExe()
 10     {
 11         // 创建编译器对象
 12         CSharpCodeProvider p = new CSharpCodeProvider();
 13         ICodeCompiler cc = p.CreateCompiler();
 14 
 15         // 设置编译参数
 16         CompilerParameters options = new CompilerParameters();
 17         options.ReferencedAssemblies.Add("System.dll");
 18         options.ReferencedAssemblies.Add("System.Data.Linq.dll");
 19         options.ReferencedAssemblies.Add("System.Data.dll");
 20         options.ReferencedAssemblies.Add("System.Core.dll");
 21 
 22         options.ReferencedAssemblies.Add("Dx.Business.dll");
 23         options.ReferencedAssemblies.Add("Dx.Model.dll");
 24         options.ReferencedAssemblies.Add("Dx.Utility.dll");
 25         options.ReferencedAssemblies.Add("Dx.Configuration.dll");
 26         options.ReferencedAssemblies.Add("Dx.IDAL.dll");
 27         options.ReferencedAssemblies.Add("Dx.DataAccess.dll");
 28         options.ReferencedAssemblies.Add("Dx.Data.dll");
 29         options.ReferencedAssemblies.Add("Dx.Framework.dll");
 30         options.ReferencedAssemblies.Add("ICSharpCode.SharpZipLib.dll");
 31         options.ReferencedAssemblies.Add("Microsoft.CSharp.dll");
 32 
 33         options.EmbeddedResources.Add("HelloWorld.exe.config"); 
 34         //options.CompilerOptions += " /target:winexe";
 35         //options.CompilerOptions += " /res:Resource1.res";
 36         //options.CompilerOptions += " /win32icon:test.ico";
 37 
 38         options.GenerateExecutable = true;
 39         options.OutputAssembly = "HelloWorld.exe";
 40 
 41         // 2. 直接指定源码字符串
 42         string code = @"
 43 using System;
 44 using System.Collections.Generic;
 45 using System.Linq;
 46 using System.Text;
 47 using System.IO;
 48 using System.Data;
 49 using System.Data.SqlClient;
 50 using System.Threading;
 51 using System.Threading.Tasks;
 52 
 53 using Dx.Data;
 54 using Dx.Business.Signal;
 55 using Dx.Model.FingerprintDatabase;
 56 using Dx.Business;
 57 using Dx.Model;
 58 using Dx.Utility;
 59 using Dx.Configuration;
 60 using Dx.Business.FingerprintDatabase.BO;
 61 using Dx.Business.ATU.Parse;
 62 using Dx.Framework.Utils;
 63 using System.Text.RegularExpressions;
 64 using ICSharpCode.SharpZipLib.GZip;
 65 using System.Diagnostics;
 66 using Dx.Business.Scheme;
 67 using Dx.Business.ATU;
 68 using Dx.Business.SiChuan;
 69 using Dx.Model.SiChuan;
 70 using System.CodeDom;
 71 using System.CodeDom.Compiler;
 72 using Microsoft.CSharp;
 73 
 74 namespace Samples
 75 {
 76 public class Class1
 77 {
 78     static void Main(string[] args)
 79     {
 80         ConfigurationManger.Init(ConfigurationType.WindowsService);
 81         EnumsRegister.Register();
 82 
 83         Dx.Business.SiChuan.GridInnerDoorAnalysis.GridInnerDoorAnalysis bo=new Dx.Business.SiChuan.GridInnerDoorAnalysis.GridInnerDoorAnalysis();
 84         bo.ComputeGridInnerDoor(args[0]);
 85     }
 86 }
 87 }";
 88         CodeSnippetCompileUnit codeSnippetCompileUnit = new CodeSnippetCompileUnit(code);
 89 
 90         // 开始编译
 91         CompilerResults compilerResults = cc.CompileAssemblyFromDom(options, codeSnippetCompileUnit);
 92 
 93         // 显示编译信息
 94         if (compilerResults.Errors.Count == 0)
 95             Console.WriteLine("{0} compiled ok!", compilerResults.CompiledAssembly.Location);
 96         else
 97         {
 98             Console.WriteLine("Complie Error:");
 99             foreach (CompilerError error in compilerResults.Errors)
100                 Console.WriteLine(" {0}", error);
101         }
102     }
103 
104     public static void SubmitTask(object state)
105     {
106         List<string> ids = state as List<string>;
107 
108         System.Threading.Tasks.Task[] taskItems = new System.Threading.Tasks.Task[ids.Count];
109 
110         for (int i = 0; i < ids.Count; i++)
111         {
112             taskItems[i] = System.Threading.Tasks.Task.Factory.StartNew((object argument) =>
113             {
114                 System.Diagnostics.Process process = new System.Diagnostics.Process();
115 
116                 process.StartInfo = new System.Diagnostics.ProcessStartInfo();
117                 process.StartInfo.CreateNoWindow = true;
118                 process.StartInfo.UseShellExecute = false;
119                 process.StartInfo.WindowStyle = System.Diagnostics.ProcessWindowStyle.Hidden;
120                 process.StartInfo.Arguments = argument.ToString();
121                 process.StartInfo.FileName = "HelloWorld.exe";
122 
123                 process.Start();
124 
125                 //Console.WriteLine(process.ProcessName);
126                 process.WaitForExit();
127                 process.Close();
128                 process.Dispose();
129             }, ids[i]);
130         }
131 
132         System.Threading.Tasks.Task.WaitAll(taskItems);
133     }
134 
135     public static void Main(string[] args)
136     {
137         ConfigurationManger.Init(ConfigurationType.WindowsService);
138         EnumsRegister.Register();
139         
140         Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
141         
142         GenerateExe();
143 
144         List<BuildingVector> allBuildingVectors = buildBO.GetAllBuildingsAsGridInnerDoorAnalysis();
145 
146         // 将 指纹库数据拷贝到GridMappingBuildings 
147         // gridMappingBuildingBO.CopyFingerPrinterDatabase();
148 
149         List<string> tasks = new List<string>();
150         int take = 25;
151         for (var i = 0; i <= allBuildingVectors.Count / take; i++)
152         {
153             tasks.Add(string.Join(",", allBuildingVectors.Skip(i * take).Take(take).Select(m => m.BuildingID).ToArray()));
154         }
155 
156         int skipCount = 0;
157 
158         while (true)
159         {
160             System.Diagnostics.Process[] processes2 = System.Diagnostics.Process.GetProcessesByName("HelloWorld");
161 
162             int maxProcessCount = 36;
163             // 如果超过了最大允许任务数据不再触发新的进程任务
164             if (processes2.Length >= maxProcessCount)
165             {
166                 Thread.Sleep(1000);
167                 continue;
168             }
169 
170             // submit task
171             int submitTaskCount = maxProcessCount - processes2.Length;
172             List<string> submitTasks = tasks.Skip(skipCount).Take(submitTaskCount).ToList();
173             
174             if(submitTasks.Count==0){
175                 break;
176             }
177             
178             System.Threading.ThreadPool.QueueUserWorkItem(SubmitTask, submitTasks);
179             Thread.Sleep(5000);
180             skipCount += submitTaskCount;
181 
182             Console.WriteLine("complete task count:{0}*25,total task count:{1}*25", skipCount, tasks.Count);
183         }
189 
190         Console.WriteLine("Complete...");
191         Console.WriteLine(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
192         Console.ReadKey();
193      }
     }