下面的日志处理我在3000个线程同时调用,写20个文件,相当于每个文件150个线程不间断大规模写.没问题,我提高到10000时,出了问题,但不是日志代码的问题,而是系统创建新的线程时报错,建到3500个线程的时候出现问题.后面我会贴出测试代码.
1)FileLogWorker
///
/// 文件日志处理类,利用队列机制，让写日志调用和日志写到文件分离，调用
/// 方将要写的日志和目标文件插入到日志队列中去就返回，
/// 然后由内置线程去写到文件里去。这里用了单例模式。
///
public class FileLogWorker
{
private Queue _logCaches = null;
private Thread _writerThread = null;
private StreamWriter _streamWriter;
public string FilePath {get;private set;}
public Thread LogWriteThread { get { return _writerThread; } }
public FileLogWorker(string FilePath)
{
this.FilePath = FilePath;
_logCaches = new Queue();
_writerThread = new Thread(new ThreadStart(WriteLogToFile));
FStop = false;
_streamWriter = new StreamWriter(FilePath,true);
_writerThread.Start();
}
private bool FStop = false;
private void WriteLogToFile()
{
Int64 theCount = 0;
while (FStop == false)
{
try
{
theCount++;
string theLogContent = null;
//队列操作时需要锁定，否则会报错.队列并不是线程安全的.
//但多个队列可以同时写.
lock (this)
{
theLogContent = _logCaches.Dequeue();
}
if (theLogContent != null && theLogContent != "")
{
_streamWriter.WriteLine(theLogContent);
_streamWriter.Flush();
LastExecTime = DateTime.Now;
}
if (theCount > 10000)
{
//GC.Collect();
theCount = 0;
}
}
catch (Exception ex)
{
SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error);
}
}
}
///
/// 向文件日志写日志内容
///
/// 日志文件名
/// 日志内容
public void WriteLogContent(string logContent)
{
lock (this)
{
_logCaches.Enqueue(logContent);
}
if (_logCaches.Count > 10000)
{
Thread.CurrentThread.Join(50);
}
}
private DateTime _LastExecTime = DateTime.Now.AddDays(-1);
public DateTime LastExecTime
{
get
{
return _LastExecTime;
}
set
{
_LastExecTime = value;
}
}
///
/// 结束日志内置线程，并关闭所有文件流。程序正常退出时调用.
///
public void CloseLogThread()
{
FStop = true;
if (_writerThread != null)
{
_writerThread.Join();
}
try
{
_streamWriter.Flush();
_streamWriter.Close();
_streamWriter.Dispose();
}
catch (Exception ex)
{
SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error);
}
}
}
2)
///
/// 文件日志处理类,利用队列机制，让写日志调用和日志写到文件分离，调用
/// 方将要写的日志和目标文件插入到日志队列中去就返回，
/// 然后由内置线程去写到文件里去。这里用了单例模式。
///
public class FileLogWriter : IDisposable
{
private class LocalLockObject
{
}
private Dictionary _Workers = null;
private Thread _writerThread = null;
static FileLogWriter()
{
Instance = new FileLogWriter();
}
private FileLogWriter()
{
_Workers = new Dictionary();
_writerThread = new Thread(new ThreadStart(Execute));
FStop = false;
_writerThread.Start();
}
private FileLogWorker GetFileLogWorker(string FullPath)
{
try
{
return _Workers[FullPath];
}
catch
{
lock (typeof(LocalLockObject))
{
if (_Workers.ContainsKey(FullPath) == false)
{
FileLogWorker theWorker = new FileLogWorker(FullPath);
_Workers.Add(FullPath, theWorker);
return theWorker;
}
else
{
return _Workers[FullPath];
}
}
}
}
private bool FStop = false;
public static FileLogWriter Instance { get; private set; }
private const int Timeout = 30;
///
/// 向文件日志写日志内容
///
/// 日志文件名
/// 日志内容
public void WriteLogFile(string fileName, string logContent)
{
FileLogWorker theWorker = GetFileLogWorker(fileName);
theWorker.WriteLogContent(logContent);
}
///
/// 清理20分钟没有发生读写的文件流。对于日志文件按天产生的非常有用。
///
public void Execute()
{
while (FStop == false)
{
try
{
lock (typeof(LocalLockObject))
{
foreach (var theItem in _Workers)
{
if (DateTime.Now.Subtract(theItem.Value.LastExecTime).Minutes > Timeout)
{
theItem.Value.CloseLogThread();
_Workers.Remove(theItem.Key);
}
}
}
Thread.Sleep(100000);
}
catch (Exception ex)
{
SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error);
}
}
}
///
/// 结束日志内置线程，并关闭所有文件流。程序正常退出时调用.
///
public void CloseAll()
{
FStop = true;
if (_writerThread != null)
{
_writerThread.Join();
}
try
{
lock (typeof(LocalLockObject))
{
if (_Workers != null)
{
foreach (var theItem in _Workers)
{
theItem.Value.CloseLogThread();
_Workers.Remove(theItem.Key);
}
}
}
}
catch (Exception ex)
{
SysAppEventWriter.WriteEvent(-1, ex.Message, System.Diagnostics.EventLogEntryType.Error);
}
}
public void Dispose()
{
try
{
CloseAll();
}
catch
{
}
}
~FileLogWriter()
{
try
{
CloseAll();
}
catch
{
}
}
}
3)SysAppEventWriter
public static class SysAppEventWriter
{
public static void WriteEvent(long EventId,string EventContent,EventLogEntryType EntryType)
{
try
{
System.Diagnostics.EventInstance theEvtInst = new System.Diagnostics.EventInstance(EventId, 0, EntryType);
System.Diagnostics.EventLog.WriteEvent(AppCfgs.CurrentAppCenterID + "KeDuoSysLogs",theEvtInst, EventContent,AppCfgs.ServiceBaseAddress);
}
catch
{
}
}
}
}
4) 文件名从这个类取，当然也可以把方法封装
///
/// 系统常量性文件名规范
///
public static class SysConstFileName
{
///
/// 系统日志文件名
///
public static string SysLogFileName(string MerchantId)
{
return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\sys" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";
}
///
/// 系统跟踪日志名
///
public static string TraceLogFileName(string MerchantId)
{
return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\tra" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";
}
///
/// 系统事件日志文件名
///
public static string EventLogFileName(string MerchantId)
{
return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\evt" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";
}
///
/// 系统事件日志文件名
///
public static string SysOptLogFileName(string MerchantId)
{
return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\opt" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";
}
///
/// 系统登录日志文件名
///
public static string SysLoginFileName(string MerchantId)
{
return System.AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "logs\\login" + MerchantId + DateTime.Now.ToString("yyyy-MM-dd") + ".log";
}
}
下面是测试出问题的时候的代码：
int theCount = 10000;
for (int i = 0; i < theCount; i++)
{
new Thread((o) =>
{
int theI = (int)o;
string file = "d:\\a"+( theI % 20) .ToString()+".txt";
for (int j = 0; j < 1000000; j++)
{
FileLogWriter.Instance.WriteLogFile(file, theI.ToString() + ":" + j.ToString());
}
FileLogWriter.Instance.WriteLogFile("d:\\a.txt", theI.ToString() + "结束!");
}).Start(i);
}
作为webApp的日志类，其实用不着这么高的并发，因为IIS的web应用，并不是多线程的，而是单线程的。
补充1:windows对单个进程所能创建的线程还是有一定限制的，否则windows的线程调度处理的压力会非常重。具体的等有机会再测试一下.
839
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
