自己最近写的一组日志处理类(支持高并发处理)

下面的日志处理我在3000个线程同时调用,写20个文件,相当于每个文件150个线程不间断大规模写.没问题,我提高到10000时,出了问题,但不是日志代码的问题,而是系统创建新的线程时报错,建到3500个线程的时候出现问题.后面我会贴出测试代码.

1)FileLogWorker

/// <summary>
    /// 文件日志处理类,利用队列机制，让写日志调用和日志写到文件分离，调用
    /// 方将要写的日志和目标文件插入到日志队列中去就返回，
    /// 然后由内置线程去写到文件里去。这里用了单例模式。
    /// </summary>
    public class FileLogWorker
    {
        private Queue<string> _logCaches = null;
        private Thread _writerThread = null;
        private StreamWriter _streamWriter;
        public string FilePath {get;private set;}
        public Thread LogWriteThread { get { return _writerThread; } }
        public FileLogWorker(string FilePath)
        {
            this.FilePath = FilePath;
            _logCaches = new Queue<string>();
            _writerThread = new Thread(new ThreadStart(WriteLogToFile));
            FStop = false;
            _streamWriter = new StreamWriter(FilePath,true);
            _writerThread.Start();
        }
        private bool FStop = false;
        private void WriteLogToFile()
        {
            Int64 theCount = 0;
            while (FStop == false)
            {
                try
                {
                       theCount++;
                        string theLogContent = null;
                        //队列操作时需要锁定，否则会报错.队列并不是线程安全的.
                        //但多个队列可以同时写.
                        lock (this)
                        {
                            theLogContent = _logCaches.Dequeue();
                        }
                        if (theLogContent != null && theLogContent != "")
                        {
                            _streamWriter.WriteLine(theLogContent);
                            _streamWriter.Flush();
                            LastExecTime = DateTime.Now;
                        }
                    if (theCount > 10000)
                    {
                        //GC.Collect();
                        theCount = 0;
                    }
                }
                catch (Exception ex)
                {
                    SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error);
                }

            }
        }
         /// <summary>
        /// 向文件日志写日志内容
        /// </summary>
        /// <param name="fileName">日志文件名</param>
        /// <param name="logContent">日志内容</param>
        public void WriteLogContent(string logContent)
        {
            lock (this)
            {
                _logCaches.Enqueue(logContent);
            }
            if (_logCaches.Count > 10000)
            {
                Thread.CurrentThread.Join(50);
            }
        }
        private DateTime _LastExecTime = DateTime.Now.AddDays(-1);
        public DateTime LastExecTime
        {
            get
            {
                return _LastExecTime;
            }
            set
            {
                _LastExecTime = value;
            }
        }
        /// <summary>
        /// 结束日志内置线程，并关闭所有文件流。程序正常退出时调用.
        /// </summary>
        public void CloseLogThread()
        {
            FStop = true;
            if (_writerThread != null)
            {
                _writerThread.Join();
            }
            try
            {
                _streamWriter.Flush();
                _streamWriter.Close();
                _streamWriter.Dispose();
            }
            catch (Exception ex)
            {
                SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error);
            }
        }
    }

2)

/// <summary>
    /// 文件日志处理类,利用队列机制，让写日志调用和日志写到文件分离，调用
    /// 方将要写的日志和目标文件插入到日志队列中去就返回，
    /// 然后由内置线程去写到文件里去。这里用了单例模式。
    /// </summary>
    public class FileLogWriter : IDisposable
    {
        private class LocalLockObject
        {
        }
        private  Dictionary<string, FileLogWorker> _Workers = null;
        private Thread _writerThread = null;
        static FileLogWriter()
        {
            Instance = new FileLogWriter();
        }
        private FileLogWriter()
        {
            _Workers = new Dictionary<string, FileLogWorker>();
            _writerThread = new Thread(new ThreadStart(Execute));
            FStop = false;
            _writerThread.Start();
        }
        private FileLogWorker GetFileLogWorker(string FullPath)
        {
            try
            {
                return _Workers[FullPath];
            }
            catch
            {
                lock (typeof(LocalLockObject))
                {
                    if (_Workers.ContainsKey(FullPath) == false)
                    {
                        FileLogWorker theWorker = new FileLogWorker(FullPath);
                        _Workers.Add(FullPath, theWorker);
                        return theWorker;
                    }
                    else
                    {
                        return _Workers[FullPath];
                    }
                }
            }
        }
       
        private bool FStop = false; 
        
        
        public static FileLogWriter Instance { get; private set; }
        private const int Timeout = 30; 
        
        /// <summary>
        /// 向文件日志写日志内容
        /// </summary>
        /// <param name="fileName">日志文件名</param>
        /// <param name="logContent">日志内容</param>
        public void WriteLogFile(string fileName, string logContent)
        {
            FileLogWorker theWorker = GetFileLogWorker(fileName);
            theWorker.WriteLogContent(logContent);
        }
        /// <summary>
        /// 清理20分钟没有发生读写的文件流。对于日志文件按天产生的非常有用。
        /// </summary>
        public void Execute()
        {
            while (FStop == false)
            {
                try
                {

                    lock (typeof(LocalLockObject))
                    {
                        foreach (var theItem in _Workers)
                        {
                            if (DateTime.Now.Subtract(theItem.Value.LastExecTime).Minutes > Timeout)
                            {
                                theItem.Value.CloseLogThread();
                                _Workers.Remove(theItem.Key);
                            }
                        }
                    }
                    Thread.Sleep(100000);
                }
                catch (Exception ex)
                {
                    SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error);
                }
            }
        }
        /// <summary>
        /// 结束日志内置线程，并关闭所有文件流。程序正常退出时调用.
        /// </summary>
        public void CloseAll()
        {
            FStop = true;
            if (_writerThread != null)
            {
                _writerThread.Join();
            }
            try
            {
                lock (typeof(LocalLockObject))
                {
                    if (_Workers != null)
                    {
                        foreach (var theItem in _Workers)
                        {
                            theItem.Value.CloseLogThread();
                            _Workers.Remove(theItem.Key);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error);
            }
        }

        public void Dispose()
        {
            try
            {
                CloseAll();
            }
            catch
            {
            }
        }
        ~FileLogWriter()
        {
            try
            {
                CloseAll();
            }
            catch
            {
            }
        }
    }

3)SysAppEventWriter

 public static class SysAppEventWriter
    {
        public static void WriteEvent(long EventId,string EventContent,EventLogEntryType EntryType)
        {
            try
            {
                System.Diagnostics.EventInstance theEvtInst = new System.Diagnostics.EventInstance(EventId, 0, EntryType);
                System.Diagnostics.EventLog.WriteEvent(AppCfgs.CurrentAppCenterID + "KeDuoSysLogs",theEvtInst, EventContent,AppCfgs.ServiceBaseAddress);
            }
            catch
            {
            }
       
        }
    }
}

4) 文件名从这个类取，当然也可以把方法封装

 /// <summary>
    /// 系统常量性文件名规范
    /// </summary>
    public static class SysConstFileName
    {
        /// <summary>
        /// 系统日志文件名
        /// </summary>
        public static string SysLogFileName(string MerchantId)
        {

            return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\sys" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";
      
        }
        /// <summary>
        /// 系统跟踪日志名
        /// </summary>
        public static string TraceLogFileName(string MerchantId)
        {

            return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\tra" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";

        }
        /// <summary>
        /// 系统事件日志文件名
        /// </summary>
        public static string EventLogFileName(string MerchantId)
        {

            return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\evt" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";

        }
        /// <summary>
        /// 系统事件日志文件名
        /// </summary>
        public static string SysOptLogFileName(string MerchantId)
        {

            return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\opt" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";
    
        }
        /// <summary>
        /// 系统登录日志文件名
        /// </summary>
        public static string SysLoginFileName(string MerchantId)
        {

            return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\login" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";
        
        }
    }

下面是测试出问题的时候的代码：

int theCount = 10000;
        
            for (int i = 0; i < theCount; i++)
            {
                new Thread((o) =>
                {
                    int theI = (int)o;
                    string file = "d:\\a"+( theI % 20) .ToString()+".txt";
                    for (int j = 0; j < 1000000; j++)
                    {
                        FileLogWriter.Instance.WriteLogFile(file, theI.ToString() + ":" + j.ToString());
                    }
                    FileLogWriter.Instance.WriteLogFile("d:\\a.txt", theI.ToString() + "结束!");
                }).Start(i);
            }

作为webApp的日志类，其实用不着这么高的并发，因为IIS的web应用，并不是多线程的，而是单线程的。

补充1:windows对单个进程所能创建的线程还是有一定限制的，否则windows的线程调度处理的压力会非常重。具体的等有机会再测试一下.