利用Opentelemetry+Loki+Temp+Granafa构建端到端的可观测平台

grafana

root@ubunt-demo:~# ll /etc/grafana/
total 60
drwxr-xr-x   3 root root  4096 Apr 24 09:24 ./
drwxr-xr-x 110 root root  4096 Apr 22 16:01 ../
-rw-r--r--   1 root root 43510 Apr 22 15:17 grafana.ini
-rw-r--r--   1 root root  2270 Jan 26 00:43 ldap.toml
drwxrwxrwx   7 root root  4096 Jan 26 00:43 provisioning/
root@ubunt-demo:~# cat /etc/grafana/grafana.ini | grep -vE '^$|^#|^;'
[paths]
[server]
protocol = http
http_addr = 127.0.0.1
http_port = 3000
root_url = %(protocol)s://http_addr/grafana/
serve_from_sub_path = true

docker  run --user root --name grafana -d --net host --restart=always --mount type=volume,source=grafana,target=/var/lib/grafana -v /etc/grafana:/etc/grafana  grafana/grafana:8.3.5

loki

cat /etc/loki/loki-config.yaml
auth_enabled: false
server:
  http_listen_port: 3100
  grpc_listen_port: 9096
  grpc_server_max_recv_msg_size: 1073741824
  grpc_server_max_send_msg_size: 1073741824

common:
  path_prefix: /loki
  storage:
    filesystem:
      chunks_directory:  /loki/chunks
      rules_directory: /loki/rules

ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s
  chunk_idle_period: 5m
  chunk_retain_period: 30s
  max_transfer_retries: 0
  max_chunk_age: 20m

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h


limits_config:
  unordered_writes: true
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h
  ingestion_rate_strategy: "global"
  ingestion_rate_mb: 30
  ingestion_burst_size_mb: 15

chunk_store_config:
  max_look_back_period: 168h

table_manager:
  retention_deletes_enabled: true
  retention_period: 168h


docker run -d --user root --restart=always -m=300M --name=loki --mount type=volume,source=loki,target=/loki -v /etc/loki:/etc/loki -p 3100:3100 -p 9095:9095 grafana/loki:2.5.0 -config.file=/etc/loki/loki-config.yaml

tempo

/etc/tempo.yaml 
auth_enabled: false
server:
  http_listen_port: 3200

distributor:
  receivers:        
    jaeger:         
      protocols:    
        thrift_http:
        grpc:       
        thrift_binary:
        thrift_compact:
    zipkin:
    otlp:
      protocols:
        http:
        grpc:
    opencensus:

ingester:
  trace_idle_period: 10s             
  max_block_bytes: 1_000_000         
  max_block_duration: 5m             

compactor:
  compaction:
    compaction_window: 1h        
    max_block_bytes: 100_000_000 
    block_retention: 1h
    compacted_block_retention: 10m

storage:
  trace:
    backend: local                     
    block:
      bloom_filter_false_positive: .05 
      index_downsample_bytes: 1000     
      encoding: zstd                   
    wal:
      path: /tmp/tempo/wal             
      encoding: snappy                 
    local:
      path: /tmp/tempo/blocks
    pool:
      max_workers: 100                 
      queue_depth: 10000

docker run -d --user root --name tempo -p 3200:3200  -p 14250:14250  --restart=always -m 100M -v /etc/tempo.yaml:/etc/tempo.yaml  -v /tmp/tempo:/tmp/tempo grafana/tempo:latest  -config.file=/etc/tempo.yaml

nginx

    location ^~ /grafana {
        proxy_http_version 1.1;
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;
        proxy_pass http://127.0.0.1:3000;
        rewrite  ^/grafana/(.*)  /$1 break;
        proxy_set_header Host $host; 
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-Proto https;
    }
    location ^~ /loki/api/v1/push {
        auth_basic           "Loki";
        auth_basic_user_file /etc/nginx/mypasswords/passwdfile;
        proxy_set_header X-Original-URI $request_uri;
        proxy_set_header  X-Scope-OrgID $http_x_scope_orgid;
        proxy_set_header remote_user  $remote_user;
        #rewrite  ^/grafana/(.*) /$1 break;
        proxy_http_version 1.1;
        proxy_pass  http://127.0.0.1:3100;
        
        }    

Opentelemetry