一、链路追踪的原理
链路追踪系统(可能)最早是由Goggle公开发布的一篇论文
《Dapper, a Large-Scale Distributed Systems Tracing Infrastructure》
翻译参考: https://zhuanlan.zhihu.com/p/38255020#%E6%A6%82%E8%A7%88
参考博文: https://baijiahao.baidu.com/s?id=1708807913437543359&wfr=spider&for=pc
总结:通过事先在日志中埋点,找出相同traceId的日志,再加上parent id和span id就可以将一条完整的请求调用链串联起来。
二、SkyAPM的使用
1、为项目添加NuGet程序包SkyAPM.Agent.AspNetCore的引用
2、在程序包管理控制台执行下面命令
dotnet tool install -g SkyAPM.DotNet.CLI
dotnet skyapm config 服务名称 192.168.0.5:11800
执行上面命令会在项目根目录添加skyapm.json文件,并添加下以内容,其中的Servers结点的IP地址根据实际情况换成自己的服务器IP
{
"SkyWalking": {
"ServiceName": "服务名称",
"Namespace": "",
"HeaderVersions": [
"sw6"
],
"Sampling": {
"SamplePer3Secs": -1,
"Percentage": -1.0
},
"Logging": {
"Level": "Debug",
"FilePath": "logs/skyapm-{Date}.log"
},
"Transport": {
"Interval": 3000,
"ProtocolVersion": "v6",
"QueueSize": 30000,
"BatchSize": 3000,
"gRPC": {
"Servers": "192.168.150.134:11800",
"Timeout": 10000,
"ConnectTimeout": 10000,
"ReportTimeout": 600000
}
}
}
}
3、修改skyapm.json文件的属性”复制到输入目录“ 修改为 ”如果较新则复制”
4、展开项目的Properties,打开launchSettings.json文件,在其中的环境变量中加入
"SKYWALKING__SERVICENAME": "服务名称"
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "SkyAPM.Agent.AspNetCore"
5、部署到IIS,在web.config中添加 environmentVariable 的配置
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<location path="." inheritInChildApplications="false">
<system.webServer>
<handlers>
<add name="aspNetCore" path="*" verb="*" modules="AspNetCoreModuleV2" resourceType="Unspecified" />
</handlers>
<aspNetCore processPath="dotnet" arguments=".\KeyingPlatform.Api.dll" stdoutLogEnabled="false" stdoutLogFile=".\logs\stdout" hostingModel="inprocess">
<environmentVariables>
<environmentVariable name="ASPNETCORE_HOSTINGSTARTUPASSEMBLIES" value="SkyAPM.Agent.AspNetCore"/>
<environmentVariable name="SKYWALKING__SERVICENAME" value="KeyingPlatform.Api"/>
</environmentVariables>
</aspNetCore>
</system.webServer>
</location>
</configuration>
三、例子分析
1、示例说明
(1)按上面SkyAPM的使用在API项目中引入SkyAPM
(2)创建两个测试项目
(3)第一个项目调用第二个项目中的接口
(4) 第二个项目中调用了其他项目的接口
代码如下:
第一个项目
[HttpGet]
public async Task<string> TraceTestMethod()
{
string body = string.Empty;
var client = new HttpClient();
var request = new HttpRequestMessage
{
Method = HttpMethod.Get,
RequestUri = new Uri("http://localhost:11700/Trace2Test/TraceTestMethod"),
};
using (var response = await client.SendAsync(request))
{
response.EnsureSuccessStatusCode();
body = await response.Content.ReadAsStringAsync();
Console.WriteLine(body);
}
return body;
}
第二个项目
/// <summary>
/// 测试代码
/// </summary>
/// <returns></returns>
[HttpGet]
public async Task< string> TraceTestMethod()
{
string body = string.Empty;
var client = new HttpClient();
var request = new HttpRequestMessage
{
Method = HttpMethod.Post,
RequestUri = new Uri("测试接口地址"),
};
using (var response = await client.SendAsync(request))
{
response.EnsureSuccessStatusCode();
body = await response.Content.ReadAsStringAsync();
Console.WriteLine(body);
}
return body;
}
2、trace结果
“6f71bfc22c46b1cf516b564af18d4dbc.24.17135989337850001”
调用第一个项目的接口,写trace时先写访问第二个项目的trace,再写访问第一个项目接口的trace
3、界面展示
用户发起请求时所带的header信息
代码发起请求时所带的header信息(接口中调用远程信息)
四、源码分析
1、源码地址
源码: https://github.com/SkyAPM/SkyAPM-dotnet/
protocol-v3下载: https://github.com/apache/skywalking-data-collect-protocol/tree/29552022b01a55ec197641f569f19c1648d49acd
注意:
2、图解
程序启动时先通过环境变量的配置加载程序集:SkyAPM.Agent.AspNetCore
1、 InstrumentationHostedService: 程序入口,调用InstrumentStartup
internal class InstrumentationHostedService : IHostedService
{
private readonly IInstrumentStartup _startup;
public InstrumentationHostedService(IInstrumentStartup startup)
{
_startup = startup;
}
public Task StartAsync(CancellationToken cancellationToken)
{
return _startup.StartAsync(cancellationToken);
}
public Task StopAsync(CancellationToken cancellationToken)
{
return _startup.StopAsync(cancellationToken);
}
}
2、 InstrumentStartup:
启动服务:譬如推送trace的报告服务SegmentReportService
foreach (var service in _services)
await service.StartAsync(cancellationToken);
trace的监听:HostingTracingDiagnosticProcessor
DiagnosticListener.AllListeners.Subscribe(_observer);
public class InstrumentStartup : IInstrumentStartup
{
private readonly TracingDiagnosticProcessorObserver _observer;
private readonly IEnumerable<IExecutionService> _services;
private readonly ILogger _logger;
public InstrumentStartup(TracingDiagnosticProcessorObserver observer, IEnumerable<IExecutionService> services, ILoggerFactory loggerFactory)
{
_observer = observer;
_services = services;
_logger = loggerFactory.CreateLogger(typeof(InstrumentStartup));
}
public async Task StartAsync(CancellationToken cancellationToken = default(CancellationToken))
{
_logger.Information("Initializing ...");
foreach (var service in _services)
await service.StartAsync(cancellationToken);
DiagnosticListener.AllListeners.Subscribe(_observer);
_logger.Information("Started SkyAPM .NET Core Agent.");
}
public async Task StopAsync(CancellationToken cancellationToken = default(CancellationToken))
{
foreach (var service in _services)
await service.StopAsync(cancellationToken);
_logger.Information("Stopped SkyAPM .NET Core Agent.");
// ReSharper disable once MethodSupportsCancellation
await Task.Delay(TimeSpan.FromSeconds(2));
}
}
3、TracingDiagnosticProcessorObserver
…………
public void OnNext(DiagnosticListener listener)
{
foreach (var diagnosticProcessor in _tracingDiagnosticProcessors.Distinct(x => x.ListenerName))
{
if (listener.Name == diagnosticProcessor.ListenerName)
{
Subscribe(listener, diagnosticProcessor);
_logger.Information(
$"Loaded diagnostic listener [{diagnosticProcessor.ListenerName}].");
}
}
}
protected virtual void Subscribe(DiagnosticListener listener,
ITracingDiagnosticProcessor tracingDiagnosticProcessor)
{
var diagnosticProcessor = new TracingDiagnosticObserver(tracingDiagnosticProcessor, _loggerFactory);
listener.Subscribe(diagnosticProcessor, diagnosticProcessor.IsEnabled);
}
…………
4、TracingDiagnosticObserver: 在执行http请求前 和 http请求后 会执行的方法
internal class TracingDiagnosticObserver : IObserver<KeyValuePair<string, object>>
{
private readonly Dictionary<string, TracingDiagnosticMethod> _methodCollection;
private readonly ILogger _logger;
public TracingDiagnosticObserver(ITracingDiagnosticProcessor tracingDiagnosticProcessor,
ILoggerFactory loggerFactory)
{
_methodCollection = new TracingDiagnosticMethodCollection(tracingDiagnosticProcessor)
.ToDictionary(method => method.DiagnosticName);
_logger = loggerFactory.CreateLogger(typeof(TracingDiagnosticObserver));
}
public bool IsEnabled(string diagnosticName)
{
return _methodCollection.ContainsKey(diagnosticName);
}
public void OnCompleted()
{
}
public void OnError(Exception error)
{
}
public void OnNext(KeyValuePair<string, object> value)
{
if (!_methodCollection.TryGetValue(value.Key, out var method))
return;
try
{
method.Invoke(value.Key, value.Value);
}
catch (Exception exception)
{
_logger.Error("Invoke diagnostic method exception.", exception);
}
}
}
5、HostingTracingDiagnosticProcessor:在执行http请求前 和 http请求后添加trace的信息,请求完成后,它trace信息写入队列,等待推送到服务器
public class HostingTracingDiagnosticProcessor : ITracingDiagnosticProcessor
{
public string ListenerName { get; } = "Microsoft.AspNetCore";
private readonly ITracingContext _tracingContext;
private readonly IEntrySegmentContextAccessor _segmentContextAccessor;
private readonly IEnumerable<IHostingDiagnosticHandler> _diagnosticHandlers;
private readonly TracingConfig _tracingConfig;
public HostingTracingDiagnosticProcessor(IEntrySegmentContextAccessor segmentContextAccessor,
ITracingContext tracingContext, IEnumerable<IHostingDiagnosticHandler> diagnosticHandlers,
IConfigAccessor configAccessor)
{
_tracingContext = tracingContext;
_diagnosticHandlers = diagnosticHandlers.Reverse();
_segmentContextAccessor = segmentContextAccessor;
_tracingConfig = configAccessor.Get<TracingConfig>();
}
/// <remarks>
/// Variable name starts with an upper case, because it's used for parameter binding. In both ASP .NET Core 2.x and 3.x we get an object in which
/// HttpContext of the current request is available under the `HttpContext` property.
/// </remarks>
[DiagnosticName("Microsoft.AspNetCore.Hosting.HttpRequestIn.Start")]
public void BeginRequest([Property] HttpContext HttpContext)
{
foreach (var handler in _diagnosticHandlers)
{
if (handler.OnlyMatch(HttpContext))
{
handler.BeginRequest(_tracingContext, HttpContext);
return;
}
}
}
/// <remarks>
/// See remarks in <see cref="BeginRequest(HttpContext)"/>.
/// </remarks>
[DiagnosticName("Microsoft.AspNetCore.Hosting.HttpRequestIn.Stop")]
public void EndRequest([Property] HttpContext HttpContext)
{
var context = _segmentContextAccessor.Context;
if (context == null)
{
return;
}
foreach (var handler in _diagnosticHandlers)
{
if (handler.OnlyMatch(HttpContext))
{
handler.EndRequest(context, HttpContext);
break;
}
}
_tracingContext.Release(context);
}
}
6、DefaultHostingDiagnosticHandler:生成 trace 信息
public class DefaultHostingDiagnosticHandler : IHostingDiagnosticHandler
{
private readonly HostingDiagnosticConfig _config;
public DefaultHostingDiagnosticHandler(IConfigAccessor configAccessor)
{
_config = configAccessor.Get<HostingDiagnosticConfig>();
}
public bool OnlyMatch(HttpContext request)
{
return true;
}
public void BeginRequest(ITracingContext tracingContext, HttpContext httpContext)
{
var context = tracingContext.CreateEntrySegmentContext(httpContext.Request.Path,
new HttpRequestCarrierHeaderCollection(httpContext.Request));
context.Span.SpanLayer = SpanLayer.HTTP;
context.Span.Component = Common.Components.ASPNETCORE;
context.Span.Peer = new StringOrIntValue(httpContext.Connection.RemoteIpAddress.ToString());
context.Span.AddTag(Tags.URL, httpContext.Request.GetDisplayUrl());
context.Span.AddTag(Tags.PATH, httpContext.Request.Path);
context.Span.AddTag(Tags.HTTP_METHOD, httpContext.Request.Method);
if(_config.CollectCookies?.Count > 0)
{
var cookies = CollectCookies(httpContext, _config.CollectCookies);
if (!string.IsNullOrEmpty(cookies))
context.Span.AddTag(Tags.HTTP_COOKIES, cookies);
}
if(_config.CollectHeaders?.Count > 0)
{
var headers = CollectHeaders(httpContext, _config.CollectHeaders);
if (!string.IsNullOrEmpty(headers))
context.Span.AddTag(Tags.HTTP_HEADERS, headers);
}
if(_config.CollectBodyContentTypes?.Count > 0)
{
var body = CollectBody(httpContext, _config.CollectBodyLengthThreshold);
if (!string.IsNullOrEmpty(body))
context.Span.AddTag(Tags.HTTP_REQUEST_BODY, body);
}
}
public void EndRequest(SegmentContext segmentContext, HttpContext httpContext)
{
var statusCode = httpContext.Response.StatusCode;
if (statusCode >= 400)
{
segmentContext.Span.ErrorOccurred();
}
segmentContext.Span.AddTag(Tags.STATUS_CODE, statusCode);
}
private string CollectCookies(HttpContext httpContext, IEnumerable<string> keys)
{
var sb = new StringBuilder();
foreach (var key in keys)
{
if (!httpContext.Request.Cookies.TryGetValue(key, out string value))
continue;
if(sb.Length > 0)
sb.Append("; ");
sb.Append(key);
sb.Append('=');
sb.Append(value);
}
return sb.ToString();
}
private string CollectHeaders(HttpContext httpContext, IEnumerable<string> keys)
{
var sb = new StringBuilder();
foreach (var key in keys)
{
if (!httpContext.Request.Headers.TryGetValue(key, out StringValues value))
continue;
if(sb.Length > 0)
sb.Append('\n');
sb.Append(key);
sb.Append(": ");
sb.Append(value);
}
return sb.ToString();
}
private string CollectBody(HttpContext httpContext, int lengthThreshold)
{
var request = httpContext.Request;
if (string.IsNullOrEmpty(httpContext.Request.ContentType)
|| httpContext.Request.ContentLength == null
|| request.ContentLength > lengthThreshold)
{
return null;
}
var contentType = new ContentType(request.ContentType);
if (!_config.CollectBodyContentTypes.Any(supportedType => contentType.MediaType == supportedType))
return null;
#if NETSTANDARD2_0
httpContext.Request.EnableRewind();
#else
httpContext.Request.EnableBuffering();
#endif
request.Body.Position = 0;
try
{
var encoding = contentType.CharSet.ToEncoding(Encoding.UTF8);
using (var reader = new StreamReader(request.Body, encoding, true, 1024, true))
{
var body = reader.ReadToEndAsync().Result;
return body;
}
}
finally
{
request.Body.Position = 0;
}
}
}
7、AsyncQueueSegmentDispatcher:trace入队、定时推送trace信息
public class AsyncQueueSegmentDispatcher : ISegmentDispatcher
{
private readonly ILogger _logger;
private readonly TransportConfig _config;
private readonly ISegmentReporter _segmentReporter;
private readonly ISegmentContextMapper _segmentContextMapper;
private readonly ConcurrentQueue<SegmentRequest> _segmentQueue;
private readonly IRuntimeEnvironment _runtimeEnvironment;
private readonly CancellationTokenSource _cancellation;
private int _offset;
public AsyncQueueSegmentDispatcher(IConfigAccessor configAccessor,
ISegmentReporter segmentReporter, IRuntimeEnvironment runtimeEnvironment,
ISegmentContextMapper segmentContextMapper, ILoggerFactory loggerFactory)
{
_segmentReporter = segmentReporter;
_segmentContextMapper = segmentContextMapper;
_runtimeEnvironment = runtimeEnvironment;
_logger = loggerFactory.CreateLogger(typeof(AsyncQueueSegmentDispatcher));
_config = configAccessor.Get<TransportConfig>();
_segmentQueue = new ConcurrentQueue<SegmentRequest>();
_cancellation = new CancellationTokenSource();
}
public bool Dispatch(SegmentContext segmentContext)
{
if (!_runtimeEnvironment.Initialized || segmentContext == null || !segmentContext.Sampled)
return false;
// todo performance optimization for ConcurrentQueue
if (_config.QueueSize < _offset || _cancellation.IsCancellationRequested)
return false;
var segment = _segmentContextMapper.Map(segmentContext);
if (segment == null)
return false;
_segmentQueue.Enqueue(segment);
Interlocked.Increment(ref _offset);
_logger.Debug($"Dispatch trace segment. [SegmentId]={segmentContext.SegmentId}.");
return true;
}
public Task Flush(CancellationToken token = default(CancellationToken))
{
// todo performance optimization for ConcurrentQueue
//var queued = _segmentQueue.Count;
//var limit = queued <= _config.PendingSegmentLimit ? queued : _config.PendingSegmentLimit;
var limit = _config.BatchSize;
var index = 0;
var segments = new List<SegmentRequest>(limit);
while (index++ < limit && _segmentQueue.TryDequeue(out var request))
{
segments.Add(request);
Interlocked.Decrement(ref _offset);
}
// send async
if (segments.Count > 0)
_segmentReporter.ReportAsync(segments, token);
Interlocked.Exchange(ref _offset, _segmentQueue.Count);
return Task.CompletedTask;
}
public void Close()
{
_cancellation.Cancel();
}
}