SkyAPM是如何实现链路追踪的

一、链路追踪的原理

链路追踪系统（可能）最早是由Goggle公开发布的一篇论文
《Dapper, a Large-Scale Distributed Systems Tracing Infrastructure》

翻译参考: https://zhuanlan.zhihu.com/p/38255020#%E6%A6%82%E8%A7%88
参考博文： https://baijiahao.baidu.com/s?id=1708807913437543359&wfr=spider&for=pc

总结：通过事先在日志中埋点，找出相同traceId的日志，再加上parent id和span id就可以将一条完整的请求调用链串联起来。

二、SkyAPM的使用

1、为项目添加NuGet程序包SkyAPM.Agent.AspNetCore的引用

2、在程序包管理控制台执行下面命令

dotnet tool install -g SkyAPM.DotNet.CLI  
dotnet skyapm config 服务名称 192.168.0.5:11800

执行上面命令会在项目根目录添加skyapm.json文件，并添加下以内容，其中的Servers结点的IP地址根据实际情况换成自己的服务器IP

{
  "SkyWalking": {
    "ServiceName": "服务名称",
    "Namespace": "",
    "HeaderVersions": [
      "sw6"
    ],
    "Sampling": {
      "SamplePer3Secs": -1,
      "Percentage": -1.0
    },
    "Logging": {
      "Level": "Debug",
      "FilePath": "logs/skyapm-{Date}.log"
    },
    "Transport": {
      "Interval": 3000,
      "ProtocolVersion": "v6",
      "QueueSize": 30000,
      "BatchSize": 3000,
      "gRPC": {
        "Servers": "192.168.150.134:11800",
        "Timeout": 10000,
        "ConnectTimeout": 10000,
        "ReportTimeout": 600000
      }
    }
  }
}

3、修改skyapm.json文件的属性”复制到输入目录“ 修改为 ”如果较新则复制”

4、展开项目的Properties，打开launchSettings.json文件，在其中的环境变量中加入

"SKYWALKING__SERVICENAME": "服务名称"  
"ASPNETCORE_HOSTINGSTARTUPASSEMBLIES": "SkyAPM.Agent.AspNetCore"

5、部署到IIS，在web.config中添加 environmentVariable 的配置

<?xml version="1.0" encoding="utf-8"?>
<configuration>
  <location path="." inheritInChildApplications="false">
    <system.webServer>
      <handlers>
        <add name="aspNetCore" path="*" verb="*" modules="AspNetCoreModuleV2" resourceType="Unspecified" />
      </handlers>
      <aspNetCore processPath="dotnet" arguments=".\KeyingPlatform.Api.dll" stdoutLogEnabled="false" stdoutLogFile=".\logs\stdout" hostingModel="inprocess">
         <environmentVariables>
          <environmentVariable name="ASPNETCORE_HOSTINGSTARTUPASSEMBLIES" value="SkyAPM.Agent.AspNetCore"/>
          <environmentVariable name="SKYWALKING__SERVICENAME" value="KeyingPlatform.Api"/>
        </environmentVariables>
      </aspNetCore>
    </system.webServer>
  </location>
</configuration>

三、例子分析

1、示例说明

（1）按上面SkyAPM的使用在API项目中引入SkyAPM
（2）创建两个测试项目
（3）第一个项目调用第二个项目中的接口
(4) 第二个项目中调用了其他项目的接口
代码如下：
第一个项目

		[HttpGet]
		public async Task<string> TraceTestMethod()
		{
			string body = string.Empty;
			var client = new HttpClient();
			var request = new HttpRequestMessage
			{
				Method = HttpMethod.Get,
				RequestUri = new Uri("http://localhost:11700/Trace2Test/TraceTestMethod"),
			};
			using (var response = await client.SendAsync(request))
			{
				response.EnsureSuccessStatusCode();
				body = await response.Content.ReadAsStringAsync();
				Console.WriteLine(body);
			}
			return body;
		}

第二个项目

		/// <summary>
		/// 测试代码
		/// </summary>
		/// <returns></returns>
		[HttpGet]
		public async Task< string> TraceTestMethod()
		{
			string body = string.Empty;
			var client = new HttpClient();
			var request = new HttpRequestMessage
			{
				Method = HttpMethod.Post,
				RequestUri = new Uri("测试接口地址"),
			};
			using (var response = await client.SendAsync(request))
			{
				response.EnsureSuccessStatusCode();
				 body = await response.Content.ReadAsStringAsync();
				Console.WriteLine(body);
			}
			return body;
		}

2、trace结果

“6f71bfc22c46b1cf516b564af18d4dbc.24.17135989337850001”

调用第一个项目的接口，写trace时先写访问第二个项目的trace,再写访问第一个项目接口的trace

3、界面展示

用户发起请求时所带的header信息

代码发起请求时所带的header信息（接口中调用远程信息）

四、源码分析

1、源码地址

源码： https://github.com/SkyAPM/SkyAPM-dotnet/
protocol-v3下载： https://github.com/apache/skywalking-data-collect-protocol/tree/29552022b01a55ec197641f569f19c1648d49acd
注意：

2、图解

程序启动时先通过环境变量的配置加载程序集：SkyAPM.Agent.AspNetCore

1、 InstrumentationHostedService: 程序入口,调用InstrumentStartup

    internal class InstrumentationHostedService : IHostedService
    {
        private readonly IInstrumentStartup _startup;

        public InstrumentationHostedService(IInstrumentStartup startup)
        {
            _startup = startup;
        }

        public Task StartAsync(CancellationToken cancellationToken)
        {
            return _startup.StartAsync(cancellationToken);
        }

        public Task StopAsync(CancellationToken cancellationToken)
        {
            return _startup.StopAsync(cancellationToken);
        }
    }

2、 InstrumentStartup：

启动服务：譬如推送trace的报告服务SegmentReportService
foreach (var service in _services)
await service.StartAsync(cancellationToken);

trace的监听：HostingTracingDiagnosticProcessor
DiagnosticListener.AllListeners.Subscribe(_observer);

    public class InstrumentStartup : IInstrumentStartup
    {
        private readonly TracingDiagnosticProcessorObserver _observer;
        private readonly IEnumerable<IExecutionService> _services;
        private readonly ILogger _logger;

        public InstrumentStartup(TracingDiagnosticProcessorObserver observer, IEnumerable<IExecutionService> services, ILoggerFactory loggerFactory)
        {
            _observer = observer;
            _services = services;
            _logger = loggerFactory.CreateLogger(typeof(InstrumentStartup));
        }

        public async Task StartAsync(CancellationToken cancellationToken = default(CancellationToken))
        {
            _logger.Information("Initializing ...");
            foreach (var service in _services)
                await service.StartAsync(cancellationToken);
            DiagnosticListener.AllListeners.Subscribe(_observer);
            _logger.Information("Started SkyAPM .NET Core Agent.");
        }

        public async Task StopAsync(CancellationToken cancellationToken = default(CancellationToken))
        {
            foreach (var service in _services)
                await service.StopAsync(cancellationToken);
            _logger.Information("Stopped SkyAPM .NET Core Agent.");
            // ReSharper disable once MethodSupportsCancellation
            await Task.Delay(TimeSpan.FromSeconds(2));
        }
    }

3、TracingDiagnosticProcessorObserver

						…………
						
    public void OnNext(DiagnosticListener listener)
    {
        foreach (var diagnosticProcessor in _tracingDiagnosticProcessors.Distinct(x => x.ListenerName))
        {
            if (listener.Name == diagnosticProcessor.ListenerName)
            {
                Subscribe(listener, diagnosticProcessor);
                _logger.Information(
                    $"Loaded diagnostic listener [{diagnosticProcessor.ListenerName}].");
            }
        }
    }

    protected virtual void Subscribe(DiagnosticListener listener,
        ITracingDiagnosticProcessor tracingDiagnosticProcessor)
    {
        var diagnosticProcessor = new TracingDiagnosticObserver(tracingDiagnosticProcessor, _loggerFactory);
        listener.Subscribe(diagnosticProcessor, diagnosticProcessor.IsEnabled);
    }


						…………

4、TracingDiagnosticObserver： 在执行http请求前 和 http请求后 会执行的方法

 internal class TracingDiagnosticObserver : IObserver<KeyValuePair<string, object>>
 {
     private readonly Dictionary<string, TracingDiagnosticMethod> _methodCollection;
     private readonly ILogger _logger;

     public TracingDiagnosticObserver(ITracingDiagnosticProcessor tracingDiagnosticProcessor,
         ILoggerFactory loggerFactory)
     {
         _methodCollection = new TracingDiagnosticMethodCollection(tracingDiagnosticProcessor)
             .ToDictionary(method => method.DiagnosticName);
         _logger = loggerFactory.CreateLogger(typeof(TracingDiagnosticObserver));
     }

     public bool IsEnabled(string diagnosticName)
     {
         return _methodCollection.ContainsKey(diagnosticName);
     }

     public void OnCompleted()
     {
     }

     public void OnError(Exception error)
     {
     }

     public void OnNext(KeyValuePair<string, object> value)
     {
         if (!_methodCollection.TryGetValue(value.Key, out var method))
             return;

         try
         {
             method.Invoke(value.Key, value.Value);
         }
         catch (Exception exception)
         {
             _logger.Error("Invoke diagnostic method exception.", exception);
         }
     }
 }

5、HostingTracingDiagnosticProcessor：在执行http请求前 和 http请求后添加trace的信息，请求完成后，它trace信息写入队列，等待推送到服务器

public class HostingTracingDiagnosticProcessor : ITracingDiagnosticProcessor
{
    public string ListenerName { get; } = "Microsoft.AspNetCore";

    private readonly ITracingContext _tracingContext;
    private readonly IEntrySegmentContextAccessor _segmentContextAccessor;
    private readonly IEnumerable<IHostingDiagnosticHandler> _diagnosticHandlers;
    private readonly TracingConfig _tracingConfig;

    public HostingTracingDiagnosticProcessor(IEntrySegmentContextAccessor segmentContextAccessor,
        ITracingContext tracingContext, IEnumerable<IHostingDiagnosticHandler> diagnosticHandlers,
        IConfigAccessor configAccessor)
    {
        _tracingContext = tracingContext;
        _diagnosticHandlers = diagnosticHandlers.Reverse();
        _segmentContextAccessor = segmentContextAccessor;
        _tracingConfig = configAccessor.Get<TracingConfig>();
    }

    /// <remarks>
    /// Variable name starts with an upper case, because it's used for parameter binding. In both ASP .NET Core 2.x and 3.x we get an object in which 
    /// HttpContext of the current request is available under the `HttpContext` property.
    /// </remarks>
    [DiagnosticName("Microsoft.AspNetCore.Hosting.HttpRequestIn.Start")]
    public void BeginRequest([Property] HttpContext HttpContext)
    {
        foreach (var handler in _diagnosticHandlers)
        {
            if (handler.OnlyMatch(HttpContext))
            {
                handler.BeginRequest(_tracingContext, HttpContext);
                return;
            }
        }
    }

    /// <remarks>
    /// See remarks in <see cref="BeginRequest(HttpContext)"/>.
    /// </remarks>
    [DiagnosticName("Microsoft.AspNetCore.Hosting.HttpRequestIn.Stop")]
    public void EndRequest([Property] HttpContext HttpContext)
    {
        var context = _segmentContextAccessor.Context;
        if (context == null)
        {
            return;
        }

        foreach (var handler in _diagnosticHandlers)
        {
            if (handler.OnlyMatch(HttpContext))
            {
                handler.EndRequest(context, HttpContext);
                break;
            }
        }

        _tracingContext.Release(context);
    }

}

6、DefaultHostingDiagnosticHandler:生成 trace 信息

    public class DefaultHostingDiagnosticHandler : IHostingDiagnosticHandler
    {
        private readonly HostingDiagnosticConfig _config;

        public DefaultHostingDiagnosticHandler(IConfigAccessor configAccessor)
        {
            _config = configAccessor.Get<HostingDiagnosticConfig>();
        }

        public bool OnlyMatch(HttpContext request)
        {
            return true;
        }

        public void BeginRequest(ITracingContext tracingContext, HttpContext httpContext)
        {
            var context = tracingContext.CreateEntrySegmentContext(httpContext.Request.Path,
                new HttpRequestCarrierHeaderCollection(httpContext.Request));
            context.Span.SpanLayer = SpanLayer.HTTP;
            context.Span.Component = Common.Components.ASPNETCORE;
            context.Span.Peer = new StringOrIntValue(httpContext.Connection.RemoteIpAddress.ToString());
            context.Span.AddTag(Tags.URL, httpContext.Request.GetDisplayUrl());
            context.Span.AddTag(Tags.PATH, httpContext.Request.Path);
            context.Span.AddTag(Tags.HTTP_METHOD, httpContext.Request.Method);

            if(_config.CollectCookies?.Count > 0)
            {
                var cookies = CollectCookies(httpContext, _config.CollectCookies);
                if (!string.IsNullOrEmpty(cookies))
                    context.Span.AddTag(Tags.HTTP_COOKIES, cookies);
            }

            if(_config.CollectHeaders?.Count > 0)
            {
                var headers = CollectHeaders(httpContext, _config.CollectHeaders);
                if (!string.IsNullOrEmpty(headers))
                    context.Span.AddTag(Tags.HTTP_HEADERS, headers);
            }

            if(_config.CollectBodyContentTypes?.Count > 0)
            {
                var body = CollectBody(httpContext, _config.CollectBodyLengthThreshold);
                if (!string.IsNullOrEmpty(body))
                    context.Span.AddTag(Tags.HTTP_REQUEST_BODY, body);
            }
        }

        public void EndRequest(SegmentContext segmentContext, HttpContext httpContext)
        {
            var statusCode = httpContext.Response.StatusCode;
            if (statusCode >= 400)
            {
                segmentContext.Span.ErrorOccurred();
            }

            segmentContext.Span.AddTag(Tags.STATUS_CODE, statusCode);
        }

        private string CollectCookies(HttpContext httpContext, IEnumerable<string> keys)
        {
            var sb = new StringBuilder();
            foreach (var key in keys)
            {
                if (!httpContext.Request.Cookies.TryGetValue(key, out string value))
                    continue;

                if(sb.Length > 0)
                    sb.Append("; ");

                sb.Append(key);
                sb.Append('=');
                sb.Append(value);
            }
            return sb.ToString();
        }

        private string CollectHeaders(HttpContext httpContext, IEnumerable<string> keys)
        {
            var sb = new StringBuilder();
            foreach (var key in keys)
            {
                if (!httpContext.Request.Headers.TryGetValue(key, out StringValues value))
                    continue;

                if(sb.Length > 0)
                    sb.Append('\n');

                sb.Append(key);
                sb.Append(": ");
                sb.Append(value);
            }
            return sb.ToString();
        }

        private string CollectBody(HttpContext httpContext, int lengthThreshold)
        {
            var request = httpContext.Request;

            if (string.IsNullOrEmpty(httpContext.Request.ContentType)
                || httpContext.Request.ContentLength == null
                || request.ContentLength > lengthThreshold)
            {
                return null;
            }

            var contentType = new ContentType(request.ContentType);
            if (!_config.CollectBodyContentTypes.Any(supportedType => contentType.MediaType == supportedType))
                return null;

#if NETSTANDARD2_0
            httpContext.Request.EnableRewind();
#else
            httpContext.Request.EnableBuffering();
#endif
            request.Body.Position = 0;
            try
            {
                var encoding = contentType.CharSet.ToEncoding(Encoding.UTF8);
                using (var reader = new StreamReader(request.Body, encoding, true, 1024, true))
                {
                    var body = reader.ReadToEndAsync().Result;
                    return body;
                }
            }
            finally
            {
                request.Body.Position = 0;
            }
        }
    }

7、AsyncQueueSegmentDispatcher：trace入队、定时推送trace信息

    public class AsyncQueueSegmentDispatcher : ISegmentDispatcher
    {
        private readonly ILogger _logger;
        private readonly TransportConfig _config;
        private readonly ISegmentReporter _segmentReporter;
        private readonly ISegmentContextMapper _segmentContextMapper;
        private readonly ConcurrentQueue<SegmentRequest> _segmentQueue;
        private readonly IRuntimeEnvironment _runtimeEnvironment;
        private readonly CancellationTokenSource _cancellation;
        private int _offset;

        public AsyncQueueSegmentDispatcher(IConfigAccessor configAccessor,
            ISegmentReporter segmentReporter, IRuntimeEnvironment runtimeEnvironment,
            ISegmentContextMapper segmentContextMapper, ILoggerFactory loggerFactory)
        {
            _segmentReporter = segmentReporter;
            _segmentContextMapper = segmentContextMapper;
            _runtimeEnvironment = runtimeEnvironment;
            _logger = loggerFactory.CreateLogger(typeof(AsyncQueueSegmentDispatcher));
            _config = configAccessor.Get<TransportConfig>();
            _segmentQueue = new ConcurrentQueue<SegmentRequest>();
            _cancellation = new CancellationTokenSource();
        }

        public bool Dispatch(SegmentContext segmentContext)
        {
            if (!_runtimeEnvironment.Initialized || segmentContext == null || !segmentContext.Sampled)
                return false;

            // todo performance optimization for ConcurrentQueue
            if (_config.QueueSize < _offset || _cancellation.IsCancellationRequested)
                return false;

            var segment = _segmentContextMapper.Map(segmentContext);

            if (segment == null)
                return false;

            _segmentQueue.Enqueue(segment);

            Interlocked.Increment(ref _offset);

            _logger.Debug($"Dispatch trace segment. [SegmentId]={segmentContext.SegmentId}.");
            return true;
        }

        public Task Flush(CancellationToken token = default(CancellationToken))
        {
            // todo performance optimization for ConcurrentQueue
            //var queued = _segmentQueue.Count;
            //var limit = queued <= _config.PendingSegmentLimit ? queued : _config.PendingSegmentLimit;
            var limit = _config.BatchSize;
            var index = 0;
            var segments = new List<SegmentRequest>(limit);
            while (index++ < limit && _segmentQueue.TryDequeue(out var request))
            {
                segments.Add(request);
                Interlocked.Decrement(ref _offset);
            }

            // send async
            if (segments.Count > 0)
                _segmentReporter.ReportAsync(segments, token);

            Interlocked.Exchange(ref _offset, _segmentQueue.Count);

            return Task.CompletedTask;
        }

        public void Close()
        {
            _cancellation.Cancel();
        }
    }