asp.net core + app.Metrics+grafana 应用监控

1.grafana&loki&InfluxDB-docker安装

loki日志，influxdb收集监控数据

version: "3"
 
networks:
  lokiNet:
    name: loki_lokiNet
    #external: true #使用已存在的网络链接
 
services:
  loki:
    image: grafana/loki:latest
    # ports: #通过nginx代理 basic认证
    #   - "3100:3100"
    volumes:
      - D:\Developments\docker\Grafana\Loki\chunks:/loki/chunks
      - D:\Developments\docker\Grafana\Loki\rules:/loki/rules
      - D:\Developments\docker\Grafana\Loki\config:/etc/loki
    command: -config.file=/etc/loki/local-config.yaml
    networks:
      - lokiNet
    environment: ## 配置环境变量 时区
      - TZ=Asia/Shanghai
      - LANG=en_US.UTF-8

  nginx:
    image: laurentbel/nginx-basic-auth
    ports:
      - "3100:80" #本机host必须为80端口
    depends_on:
      - loki
    environment: 
      - FORWARD_HOST=loki
      - FORWARD_PORT=3100
      - BASIC_USERNAME=loki
      - BASIC_PASSWORD=loki123
    networks:
      - lokiNet
 
  promtail:
    image: grafana/promtail:latest
    volumes:
      - D:\Developments\docker\Grafana\Loki\promtail\log:/var/log
      - D:\Developments\docker\Grafana\Loki\config:/etc/promtail/
    command: -config.file=/etc/promtail/promtail-config.yml
    networks:
      - lokiNet
    environment: ## 配置环境变量 时区
      - TZ=Asia/Shanghai
      - LANG=en_US.UTF-8
 
  grafana:
    #grafana-cli plugins install grafana-piechart-panel
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    networks:
      - lokiNet
    environment: ## 配置环境变量，设置 Grafana 的默认管理员用户名/密码 
      - GF_SECURITY_ADMIN_USER=admin
      - GF_SECURITY_ADMIN_PASSWORD=admin123
      - GF_USERS_ALLOW_SIGN_UP=false
      - TZ=Asia/Shanghai
      - LANG=en_US.UTF-8
      
  influxdb:
    image: influxdb:latest
    ports:
      - "8086:8086"
    volumes:
      # 数据库文件路径
      - D:\Developments\docker\Grafana\Loki\Influxdb\Data:/root/influxdb/data
      - D:\Developments\docker\Grafana\Loki\Influxdb\Data\influxdb:/var/lib/influxdb
      - D:\Developments\docker\Grafana\Loki\Influxdb\Data\influxdb2:/var/lib/influxdb2
      # 配置文件路径
      # - D:\Developments\docker\Grafana\Loki\Influxdb:/etc/influxdb/influxdb.conf
    networks:
      - lokiNet
    environment:
      - TZ=Asia/Shanghai
      - LANG=en_US.UTF-8
      # 设置默认的账户密码 docker安装 MODE=upgrade必须设置
      # 且必须指明 config、blot、engine的路径
      - DOCKER_INFLUXDB_INIT_MODE=upgrade
      - DOCKER_INFLUXDB_INIT_USERNAME=admin
      - DOCKER_INFLUXDB_INIT_PASSWORD=admin123
      - DOCKER_INFLUXDB_INIT_ORG=finch-org
      - DOCKER_INFLUXDB_INIT_BUCKET=finchdb
      - DOCKER_INFLUXDB_INIT_UPGRADE_V1_CONFIG=/root/influxdb/influxdb.conf
      - DOCKER_INFLUXDB_CONFIG_PATH=/var/lib/influxdb2/config.conf
      - DOCKER_INFLUXDB_BOLT_PATH=/var/lib/influxdb2/influxdb.bolt
      - DOCKER_INFLUXDB_ENGINE_PATH=/var/lib/influxdb2/engine
 
  # volumes:
  #   loki_chunks:
  #   loki_rules:
  #   promtail_log:

------- 创建 链接配置 账户为 第一次登录时配置的账户密码-------------------------------------------

influx config create \

  -n MyQLCnf \

  -u http://localhost:8086 \

  -p finch:finch123 \

  -o finch.inc \

  --active

------- 创建 v1 版本的账户 -c 使用账户登录并绑定至v1版本新账户-------------------------------------------

influx v1 auth create \

  --read-bucket f93a8aa5de5d663d \

  --write-bucket f93a8aa5de5d663d \

  --username michael \

  --password michael123 \

  -c MyQLCnf

------ 绑定 InfluxQL需要数据库和保留策略（DBRP）组合才能查询数据-----------------------------------

influx v1 dbrp create \

  --db finchdb \

  --rp finchdb-rp \

  --bucket-id f93a8aa5de5d663d \

  --org finch.inc \

  --default \

  -c MyQLCnf

 

-------local-config.yaml-------------------
auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096

common:
  instance_addr: 127.0.0.1
  path_prefix: /tmp/loki
  storage:
    filesystem:
      chunks_directory: /tmp/loki/chunks
      rules_directory: /tmp/loki/rules
  replication_factor: 1
  ring:
    kvstore:
      store: inmemory

query_range:
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 100

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093

# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
#
# Statistics help us better understand how Loki is used, and they show us performance
# levels for most users. This helps us prioritize features and documentation.
# For more information on what's sent, look at
# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
# Refer to the buildReport method to see what goes into a report.
#
# If you would like to disable reporting, uncomment the following lines:
#analytics:
#  reporting_enabled: false

---------loki-local-config.yaml-----------------
server:
  http_listen_port: 9080
  grpc_listen_port: 9081

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://loki:3100/loki/api/v1/push

scrape_configs:
- job_name: system
  static_configs:
  - targets:
      - localhost
    labels:
      job: varlogs
      __path__: /var/log/*log

 

2.asp.net core

2.1 app.metrics 配置

appsettings

{
    "InfluxDbConn": {
    "IsOpen": true,
    "DatabaseName": "finchdb",
    "ConnectionString": "%InfluxDbUrl%",
    "UserName": "finch",
    "Password": "admin123"
    //Grafana-import-Dashboard http://localhost:5000/dashboards/2125、https://grafana.com/dashboards/2125仅支持influxQL版本
    //只支持 influx-v1版本，账户需要做如下操作
    //influx v1 auth create --username finch --password admin123 --org-id 0fe8f98223d4fe41 -t "_MrwnRBy68aYB21hI-qnxTqb2C8zsMfVy92tHqXHercAbY50EOZppKLZpeloP4H3I90aen-kHBjZAU-lVmaYOA==" --write-bucket c1d32947f2cd67e5 --read-bucket c1d32947f2cd67e5
    //influx v1 dbrp create --org-id 0fe8f98223d4fe41 -t "_MrwnRBy68aYB21hI-qnxTqb2C8zsMfVy92tHqXHercAbY50EOZppKLZpeloP4H3I90aen-kHBjZAU-lVmaYOA==" --bucket-id c1d32947f2cd67e5 --db app_metrics --rp autogen
  },
  "MetricsOptions": {
    //未设置时，使用这个ContextName
    "DefaultContextLabel": "MyAppMetrics",
    "GlobalTags": {
      "MyApp": "DockerApp",
      "MyEnv": "Development"
    },
    "Enabled": true,
    "ReportingEnabled": true
  },
  "MetricsWebTrackingOptions": {
    "ApdexTrackingEnabled": true,
    "ApdexTSeconds": 0.1,
    "IgnoredHttpStatusCodes": [ 404 ],
    "IgnoredRoutesRegexPatterns": [],
    "OAuth2TrackingEnabled": true
  },
  "MetricEndpointsOptions": {
    "MetricsEndpointEnabled": true,
    "MetricsTextEndpointEnabled": true,
    "EnvironmentInfoEndpointEnabled": true
  }
}

startup

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using App.Metrics;
using App.Metrics.Extensions.Configuration;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.HttpsPolicy;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Serilog;

namespace WebApplication2
{
    public class Startup
    {
        public Startup(IConfiguration configuration)
        {
            Configuration = configuration;
        }

        public IConfiguration Configuration { get; }

        // This method gets called by the runtime. Use this method to add services to the container.
        public void ConfigureServices(IServiceCollection services)
        {
            services.AddControllersWithViews();
            services.AddSingleton<RedisHelp.RedisHelper>();

            services.Configure<KestrelServerOptions>(options =>
            {
                options.AllowSynchronousIO = true;
            });
            #region 健康检查&检查发布者

            //https://docs.microsoft.com/zh-cn/aspnet/core/host-and-deploy/health-checks?view=aspnetcore-3.1#health-check-publisher
            services.Configure<HealthCheckPublisherOptions>(options =>
            {
                options.Period = TimeSpan.FromSeconds(30);
                options.Delay = TimeSpan.FromSeconds(2);
                options.Predicate = healthCheck => true;

            });

            services.AddSingleton<IHealthCheckPublisher, SampleHealthCheckPublisher>(); 
            services.AddSingleton<MemoryHealthCheck>();
            services.AddHealthChecks().AddMemoryHealthCheck(
                    "webAppHealth",
                    failureStatus: HealthStatus.Degraded,
                    tags: new[] { "ready" });

            #endregion

            #region AppMetrics

            /*  https://www.jianshu.com/p/c6a8a6ac15e1
             *  GaugeOptions（仪表盘）、CounterOptions（计数器）、MeterOptions（计量器）、
             *  HistogramOptions（柱状图）、TimerOptions（时间线）、ApdexOptions（性能度量）
             */
            bool isOpenMetrics = Convert.ToBoolean(Configuration["MetricsOptions:ReportingEnabled"]);
            if (isOpenMetrics)
            {
                string database = Configuration["InfluxDbConn:DatabaseName"];
                string connStr = Configuration["InfluxDbConn:ConnectionString"];
                var uri = new Uri(Environment.ExpandEnvironmentVariables(connStr));
                string app = Configuration["InfluxDbConn:App"];
                string env = Configuration["InfluxDbConn:Env"];
                string username = Configuration["InfluxDbConn:UserName"];
                string password = Configuration["InfluxDbConn:Password"];
                var metrics = AppMetrics.CreateDefaultBuilder()
                .Configuration.ReadFrom(Configuration)
                //.Configuration.Configure(options =>
                //{
                //    //options.AddAppTag(app);//修改默认App值
                //    //options.AddEnvTag(env);//修改默认Env值
                //})
                .Report.ToInfluxDb(options =>
                {
                    options.InfluxDb.BaseUri = uri;
                    options.InfluxDb.Database = database;
                    options.InfluxDb.UserName = username;
                    options.InfluxDb.Password = password;
                    options.HttpPolicy.BackoffPeriod = TimeSpan.FromSeconds(30);
                    options.HttpPolicy.FailuresBeforeBackoff = 5;
                    options.HttpPolicy.Timeout = TimeSpan.FromSeconds(10);
                    options.FlushInterval = TimeSpan.FromSeconds(5);
                }).Build();
                //追踪
                services.AddMetricsTrackingMiddleware();
                //报告存储
                services.AddMetricsReportingHostedService();
                //终结点
                services.AddMetricsEndpoints();
                services.AddAppMetricsHealthPublishing();
                services.AddMetrics(metrics);
                //Register all available collectors
                services.AddAppMetricsCollectors();
                //services.AddAppMetricsGcEventsMetricsCollector();
                //services.AddAppMetricsSystemMetricsCollector();
            }

            #endregion
        }

        // This method gets called by the runtime. Use this method to configure the HTTP request pipeline.
        public void Configure(IApplicationBuilder app, IWebHostEnvironment env, ILoggerFactory loggerFactory)
        {
            //redis实例
            var redis = app.ApplicationServices.GetService<RedisHelp.RedisHelper>();
            redis.SubscribeKeyExpire();

            if (env.IsDevelopment())
            {
                app.UseDeveloperExceptionPage();
            }
            else
            {
                app.UseExceptionHandler("/Home/Error");
                // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
                app.UseHsts();
            }
            loggerFactory.AddLog4Net();
            var logger = new LoggerConfiguration()
            .ReadFrom.Configuration(Configuration)
            .CreateLogger();
            loggerFactory.AddSerilog(logger);

            #region AppMetrics

            bool isOpenMetrics = Convert.ToBoolean(Configuration["MetricsOptions:Enabled"]);
            if (isOpenMetrics)
            {
                app.UseMetricsAllMiddleware();
                app.UseMetricsAllEndpoints();
                //手动设置MetricsEndpoint
                //app.UseMetricsEndpoint();
                //app.UseMetricsTextEndpoint();
                //app.UseEnvInfoEndpoint();
                //手动添加中间件
                //app.UseMetricsActiveRequestMiddleware();
                //app.UseMetricsErrorTrackingMiddleware();
                //app.UseMetricsApdexTrackingMiddleware();
                //app.UseMetricsOAuth2TrackingMiddleware();
                //app.UseMetricsPostAndPutSizeTrackingMiddleware();
                //app.UseMetricsRequestTrackingMiddleware();
            }

            #endregion

            //app.UseHttpsRedirection();
            app.UseStaticFiles();

            app.UseRouting();

            app.UseAuthorization();

            app.UseEndpoints(endpoints =>
            {
                endpoints.MapControllerRoute(
                    name: "default",
                    pattern: "{controller=Home}/{action=Index}/{id?}");
                endpoints.MapHealthChecks("/health");
            });
        }
    }

    public class SampleHealthCheckPublisher : IHealthCheckPublisher
    {
        private readonly Microsoft.Extensions.Logging.ILogger _logger;

        public SampleHealthCheckPublisher(ILogger<SampleHealthCheckPublisher> logger)
        {
            _logger = logger;
        }

        public Task PublishAsync(HealthReport report, CancellationToken cancellationToken)
        {
            if (report.Status == HealthStatus.Healthy)
            {
                _logger.LogInformation("{Timestamp} Readiness Probe Status: {Result}",
                    DateTime.UtcNow, report.Status);
            }
            else
            {
                _logger.LogError("{Timestamp} Readiness Probe Status: {Result}",
                    DateTime.UtcNow, report.Status);
            }

            cancellationToken.ThrowIfCancellationRequested();

            return Task.CompletedTask;
        }
    }

    public class MemoryHealthCheck : IHealthCheck
    {
        private readonly IOptionsMonitor<MemoryCheckOptions> _options;

        public MemoryHealthCheck(IOptionsMonitor<MemoryCheckOptions> options)
        {
            _options = options;
        }

        public string Name => "memory_check";

        public Task<HealthCheckResult> CheckHealthAsync(
            HealthCheckContext context,
            CancellationToken cancellationToken = default(CancellationToken))
        {
            var options = _options.Get(context.Registration.Name);

            // Include GC information in the reported diagnostics.
            var allocated = GC.GetTotalMemory(forceFullCollection: false);
            var data = new Dictionary<string, object>()
            {
                { "AllocatedBytes", allocated },
                { "Gen0Collections", GC.CollectionCount(0) },
                { "Gen1Collections", GC.CollectionCount(1) },
                { "Gen2Collections", GC.CollectionCount(2) },
            };
            var status = (allocated < options.Threshold) ?
                HealthStatus.Healthy : context.Registration.FailureStatus;

            return Task.FromResult(new HealthCheckResult(
                status,
                description: "Reports degraded status if allocated bytes " +
                    $">= {options.Threshold} bytes.",
                exception: null,
                data: data));
        }
    }
    // </snippet1>

    // <snippet2>
    public static class GCInfoHealthCheckBuilderExtensions
    {
        public static IHealthChecksBuilder AddMemoryHealthCheck(
            this IHealthChecksBuilder builder,
            string name,
            HealthStatus? failureStatus = null,
            IEnumerable<string> tags = null,
            long? thresholdInBytes = null)
        {
            // Register a check of type GCInfo.
            builder.AddCheck<MemoryHealthCheck>(
                name, failureStatus ?? HealthStatus.Degraded, tags);

            // Configure named options to pass the threshold into the check.
            if (thresholdInBytes.HasValue)
            {
                builder.Services.Configure<MemoryCheckOptions>(name, options =>
                {
                    options.Threshold = thresholdInBytes.Value;
                });
            }

            return builder;
        }
    }
    // </snippet2>

    // <snippet3>
    public class MemoryCheckOptions
    {
        // Failure threshold (in bytes)
        public long Threshold { get; set; } = 1024L * 1024L * 1024L;
    }
}

2.2 docker-compose

version: '3.3'

services:
  web_app2:
    # https://www.runoob.com/docker/docker-compose.html
    #指定与服务的部署和运行有关的配置。只在 swarm 模式下才会有用。mode：。
    #deploy:
    #  指定服务提供的模式
    #  # replicated：复制服务，复制指定服务到集群的机器上。
    #  # global：全局服务，服务将部署至集群的每个节点。
    #  mode：replicated
    #  replicas: 6
    #  #访问集群服务的方式
    #  # vip:Docker 集群服务一个对外的虚拟 ip。所有的请求都会通过这个虚拟 ip 到达集群服务内部的机器。
    #  # dnsrr:DNS 轮询（DNSRR）。所有的请求会自动轮询获取到集群 ip 列表中的一个 ip 地址
    #  endpoint_mode: dnsrr
    #  labels: 
    #    description: "This redis service label"
    #  resources:
    #    limits:
    #      cpus: '0.50'
    #      memory: 50M
    #    reservations:
    #      cpus: '0.25'
    #      memory: 20M
    #  restart_policy:
    #    condition: on-failure
    #    delay: 5s
    #    max_attempts: 3
    #    window: 120s
    container_name: TestDockerApp
    # no：是默认的重启策略，在任何情况下都不会重启容器。
    # always：容器总是重新启动。
    # on-failure：在容器非正常退出时（退出状态非0），才会重启容器。
    # unless-stopped：在容器退出时总是重启容器，但是不考虑在Docker守护进程启动时就已经停止了的容器
    restart: always
    build:
      context: .
      dockerfile: Dockerfile
    image: finch/testdockerapp
    ports:
      - '8000:6000'
      - '8001:6001'
    volumes:
      - D:\Data\certificate:/app/test/app2/certificate
      - D:\Data\Log:/app/test/app2/Log
      #可以使用 具体的值
      #- type: bind
      #  source: D:\certificate
      #  target: /app/test/app2/certificate
      #  read_only: false
      #  bind:
      #    propagation: rslave
      #- type: bind
      #  source: D:\App2\log
      #  target: /app/test/app2/Log
      #  read_only: true
      #  bind:
      #    propagation: rslave
    # State a dependancy on Redis working
    depends_on:
      - redis
    #连接后直接按services名称作为域名
    #links:
    #  - redis
    networks:
      my_network:
      loki_lokiNet: #与其他容器的bridge网络链接需要指定一下
        aliases:
          - myloki

      #my_network:
      # ipv4_address: 192.168.0.10
      # The Application needs a connection string for Redis, this just needs to be the Redis Service name as defined below
      # Pass it in as an Environmental Variable
    environment:
      # - TZ=Asia/Shanghai
      # - LANG=en_US.UTF-8
      - RedisConnection=docker-redis:6379,password=Michael,allowAdmin=true,connectTimeout=1000,connectRetry=3
      - GrafanaLokiUrl=http://loki:3100 #loki_lokiNet下的 容器名称
      - InfluxDbUrl=http://influxdb:8086 #loki_lokiNet下的 容器名称
      - SeriLogPath=Seri_Log
      - ASPNETCORE_HTTPS_PORT=8001
    privileged: true    #环境变量
  redis:
    container_name: 'myredis-docker'
    image: 'redis'
    restart: always
    ports:
      - 6381:6379
    networks:
      # - my_network
      #可以使用别名（默认Services名称）
      my_network:
        aliases:
          - docker-redis
    #redis数据存储到卷（不受compose up/down影响）
    volumes:
      - redis_data:/data
      - D:\Data\redis\docker\redis.conf:/etc/redis/redis.conf
      - D:\Data\redis\docker\Log:/logs
    environment:
      - TZ=Asia/Shanghai
      - LANG=en_US.UTF-8
    privileged: true    #环境变量
    #命令--requirepass Michael 
    command: redis-server /etc/redis/redis.conf
#全局Volumn卷，删除Container和image，不会删除卷
volumes:
  redis_data:

networks:
  my_network:
    name: mynetwork
    driver: bridge
  loki_lokiNet:
    name: loki_lokiNet
    external: true #使用已存在的网络链接

3.grafana配置

3.1 添加数据源 loki&influxdb（infuxQL暂不支持flux）

 3.2增加dashboard 2125-webMonitoring 12616-GC&SystemMonitoring 2137-OAuth2Monitoring

 3.3 结果展示

参考：Apdex :: App MetricsUpgrade from InfluxDB 1.x to 2.1 with Docker | InfluxDB OSS 2.1 Documentation
AppMetrics简单理解 - 简书