OpenTelemetry 试玩

https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/examples/demo
https://pkg.go.dev/github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanmetricsprocessor#section-readme
https://github.com/open-telemetry/opentelemetry-demo/blob/main/docker-compose.yml
https://github.com/laziobird/otel-collector-java/

服务器IP	部暑角色
192.168.11.100	elasticsearch 、jaeger-all-in-one、otel-collector、 hotrod、spark dependencies、promethues grafana

jaeger组件介绍：
elasticsearch : jaeger 存储器
jaeger-all-in-one:
spark denpendencies: 对spans进行集合，生成依赖图
promethus: 监控
grafana：图显
otel-collector-contrib:

一、docker部暑
。。。

二、elasticsearch
2.1、集群证书生成,生成elastic-certificates.p12证书（此步要手动执行确认）

```bash
mkdir -p /data/elasticsearch/{config,logs,data}/
mkdir -p /data/elasticsearch/config/certs/

chown 1000:root /data/elasticsearch/{config,logs,data}
docker run -it --rm \
-v /data/elasticsearch/config/:/usr/share/elasticsearch/config/ \
elasticsearch:7.17.6 bash  
#以下需要手动执行
bin/elasticsearch-certutil ca  -s --pass '' --days 10000 --out elastic-stack-ca.p12

bin/elasticsearch-certutil cert -s --ca-pass '' --pass '' --days 5000 --ca elastic-stack-ca.p12  --out  elastic-certificates.p12

bin/elasticsearch-certutil cert -s --ca-pass '' --pass '' --days 5000 --ca elastic-stack-ca.p12  -name 'CN=szgd,OU=elastic Team,DC=digitalgd,DC=com' --out elastic-https.p12  --ip '192.168.11.100,127.0.0.1'  --dns *.digitalgd.com

#创建https证书（filebeat、kibana使用）
#./bin/elasticsearch-certutil cert   -s --ca-pass '' --pass '' --days 5000 --ca elastic-stack-ca.p12  -name "CN=szgd,OU=elastic Team,DC=digitalgd,DC=com" --out client.p12  --ip "192.168.11.100,192.168.11.101,192.168.11.102,127.0.0.1"  --dns "*.digitalgd.com"

openssl pkcs12 -in elastic-https.p12 -nocerts -nodes -password pass: > client.key
openssl pkcs12 -in elastic-https.p12 -clcerts -nokeys -password pass: > client.cer
openssl pkcs12 -in elastic-https.p12 -cacerts -nokeys -chain -password pass: > client-ca.cer


mv elastic-* config/certs
mv client* config/certs
chown -R 1000:root config
exit

2.2 准备elasticsearch.yml

mkdir -p /data/elasticsearch/{config,data}
cat > /data/elasticsearch/config/elasticsearch.yml << 'EOF'
cluster.name: smartgate-cluster
discovery.seed_hosts: 192.168.11.100
cluster.initial_master_nodes: 192.168.11.100
network.host: 192.168.11.100

#增加了写队列的大小
thread_pool.write.queue_size: 1000

#锁定内存
bootstrap.memory_lock: true
bootstrap.system_call_filter: false

xpack.license.self_generated.type: basic
xpack.ml.enabled: false
xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: "certificate"
xpack.security.transport.ssl.keystore.path: "certs/elastic-certificates.p12"
xpack.security.transport.ssl.truststore.path: "certs/elastic-certificates.p12"
xpack.security.enabled: true

xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: certs/elastic-https.p12
xpack.security.http.ssl.truststore.path: certs/elastic-https.p12
xpack.security.http.ssl.client_authentication: optional
xpack.security.authc.realms.pki.pki1.order: 1

node.roles: ['master','data','ingest','remote_cluster_client']
node.attr.box_type: hot
node.attr.disk_type: sas
node.name: 192.168.11.100

http.port: 9200
http.cors.allow-origin: "*"
http.cors.allow-headers: Authorization
http.cors.enabled: true
http.host: "192.168.11.100,127.0.0.1"
transport.host: "192.168.11.100,127.0.0.1"
#elastic agent 启动需要
#xpack.security.authc.api_key.enabled: true

ingest.geoip.downloader.enabled: false
EOF

cat >/data/elasticsearch/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
dockerd --iptables=false >/dev/nul 2>&1 &
sleep 1
docker start elasticsearch >/dev/nul 2>&1
if [ "$?" == "0" ]
then
docker rm elasticsearch -f
fi
sleep 1
docker start elasticsearch >/dev/nul 2>&1
if [ "$?" != "0" ]
then

echo "run elasticsearch"

docker run -d \
--restart=always \
--name elasticsearch \
--network host \
--privileged \
--ulimit memlock=-1:-1 \
--ulimit nofile=65536:65536 \
-e ELASTIC_PASSWORD=xxxxxxxx \
-e KIBANA_PASSWORD=xxxxxxxx \
-e "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
-v /etc/localtime:/etc/localtime \
-v `pwd`/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml \
-v `pwd`/config/certs/:/usr/share/elasticsearch/config/certs \
-v `pwd`/data/:/usr/share/elasticsearch/data/ \
-v `pwd`/logs/:/usr/share/elasticsearch/logs/  \
elasticsearch:7.17.6
fi
EOF

bash /data/elasticsearch/start.sh

2.3 验证es

curl -u elastic:xxxxxxxx  https://192.168.11.100:9200/ -k
{
  "name" : "192.168.11.100",
  "cluster_name" : "smartgate-cluster",
  "cluster_uuid" : "WXsvx4dpTVi5o27Zdlh5xg",
  "version" : {
    "number" : "7.17.6",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "f65e9d338dc1d07b642e14a27f338990148ee5b6",
    "build_date" : "2022-08-23T11:08:48.893373482Z",
    "build_snapshot" : false,
    "lucene_version" : "8.11.1",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

#开放firewalld端口

firewall-cmd --permanent --add-port=9200/tcp
firewall-cmd --permanent --add-port=9300/tcp
firewall-cmd --reload

三、prometheus

#创建prometheus工作目录
mkdir /data/prometheus/{data,conf,conf/rules,conf/sd_config} -p
chown -R  65534:65534 /data/prometheus/data

#promethes配置文件
cat > /data/prometheus/conf/prometheus.yml << 'EOF'
global:
  evaluation_interval: 30s
  scrape_interval: 5s
scrape_configs:
- job_name: otel
  honor_labels: true
  static_configs:
  - targets:
    - '192.168.11.100:9464'
- job_name: otel-collector
  static_configs:
  - targets:
    - '192.168.11.100:8888'
EOF

#启动脚本
cat > /data/prometheus/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
docker rm -f prometheus 

docker run -d \
--name prometheus \
--restart=always \
-p 9090:9090 \
-v /data/prometheus/conf/prometheus.yml:/etc/prometheus/prometheus.yml  \
-v /data/prometheus/conf/rules:/etc/prometheus/rules \
-v /data/prometheus/conf/sd_config:/etc/prometheus/sd_config \
-v /data/prometheus/data:/data/prometheus \
-v /etc/localtime:/etc/localtime:ro \
prom/prometheus:v2.28.0 \
--web.read-timeout=5m \
--config.file=/etc/prometheus/prometheus.yml \
--storage.tsdb.path=/data/prometheus \
--web.max-connections=512 \
--storage.tsdb.retention=30d \
--query.timeout=2m \
--web.enable-lifecycle  \
--web.listen-address=:9090  \
--web.enable-admin-api \
--enable-feature=remote-write-receiver
EOF
bash /data/prometheus/start.sh

#开放firewalld端口

firewall-cmd --permanent --add-port=9090/tcp
firewall-cmd --reload

验证地址

http://192.168.11.100:9090

四、grafana

path=/data/grafana
mkdir ${path}/data -p
chown -R 472.472 ${path}/data

cat >${path}/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
docker rm -f grafana 

docker run -d \
--name=grafana \
--restart always \
--user 472 \
-p 3000:3000 \
-e "GF_SECURITY_ADMIN_USER=admin" \
-e "GF_SECURITY_ADMIN_PASSWORD=xxxxxxxx" \
-e "GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel,grafana-clickhouse-datasource" \
-e "GF_SERVER_SERVE_FROM_SUB_PATH=true" \
-e "GF_SERVER_ROOT_URL=http://192.168.11.100:3000/grafana/" \
-v /etc/localtime:/etc/localtime:ro \
-v /data/grafana/data:/var/lib/grafana \
grafana/grafana:9.3.6
EOF

bash ${path}/start.sh

http://192.168.11.100:3000/grafana/
帐密： admin / xxxxxxxx

集成prometheus

导入以下两个dashboard
ID: 18309

ID: 18264

ID: 15983

ID: 19419

四、jaeger(all-in-one)

path=/data/jaeger
mkdir ${path} -p

#把elasticsearch 生成证书copy到 jaeger目录下
cp /data/elasticsearch/config/certs /data/jaeger/ -rf


cat >${path}/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
docker rm -f jaeger

docker run -d \
--name jaeger \
-e COLLECTOR_OTLP_ENABLED=true \
-p 5775:5775/udp \
-p 6831:6831/udp \
-p 6832:6832/udp \
-p 16686:16686 \
-p 14250:14250 \
-v /etc/localtime:/etc/localtime \
-v `pwd`/certs:/go/certs \
-e SPAN_STORAGE_TYPE=elasticsearch \
-e ES_USERNAME=elastic \
-e ES_PASSWORD=xxxxxxxx \
-e ES_SERVER_URLS=https://192.168.11.100:9200 \
-e ES_TLS_ENABLED=true \
-e ES_TLS_CA=/go/certs/client-ca.cer \
-e METRICS_STORAGE_TYPE=prometheus \
-e PROMETHEUS_SERVER_URL=http://192.168.11.100:9090 \
-e PROMETHEUS_TLS_ENABLED=false \
jaegertracing/all-in-one:1.45
EOF

bash ${path}/start.sh

#验证地址

http://192.168.11.100:16686/search

五、otel-collector

path=/data/otel
mkdir ${path} -p

cat >${path}/otel-config.yaml << 'EOF'
receivers:
  jaeger:
    protocols:
      thrift_http:
        endpoint: :14268

  # Dummy receiver that's never used, because a pipeline is required to have one.
  otlp/spanmetrics:
    protocols:
      grpc:
        endpoint: "localhost:12345"

  otlp:
    protocols:
      grpc:
        endpoint: :4317
      http:
        endpoint: :4318

  prometheus:
    config:
      scrape_configs:
      - job_name: 'ratelimiter'
        scrape_interval: 15s
        static_configs:
        - targets: [ "0.0.0.0:8889" ]
exporters:
  jaeger:
    endpoint: 192.168.11.100:14250
    tls:
      insecure: true
  prometheus:
    endpoint: "0.0.0.0:8889"
  prometheusremotewrite:
    endpoint: http://192.168.11.100:9090/api/v1/write
    target_info:
       enabled: true
processors:
  batch:
  spanmetrics:
    metrics_exporter: prometheus
    latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms]
    dimensions:
      - name: http.method
        default: GET
      - name: http.status_code
    dimensions_cache_size: 1000
    aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE"
    metrics_flush_interval: 15s
extensions:
  health_check:
  pprof:
    endpoint: :1888
  zpages:
    endpoint: :55679

service:
  extensions: [pprof, zpages, health_check]
  pipelines:
    traces:
      receivers: [jaeger,otlp]
      processors: [spanmetrics, batch]
      exporters: [jaeger]

    metrics/spanmetrics:
      receivers: [otlp/spanmetrics]
      exporters: [prometheus]

    metrics:
      receivers: [prometheus]
      exporters: [prometheusremotewrite]
EOF


cat >${path}/start.sh << 'EOF'
#!/bin/bash
cd `dirname $0`
docker rm -f otel

docker run  -itd \
--restart=always \
--name otel \
-p 1888:1888 \
-p 8888:8888 \
-p 8889:8889 \
-p 13133:13133 \
-p 4317:4317 \
-p 55679:55679 \
-p 12345:12345 \
-p 14268:14268 \
-v /etc/localtime:/etc/localtime \
-v `pwd`/otel-config.yaml:/etc/otel-collector-config.yaml \
otel/opentelemetry-collector-contrib:0.78.0 \
--config=/etc/otel-collector-config.yaml

#      - "1888:1888"   # pprof extension
#      - "8888:8888"   # Prometheus metrics exposed by the collector
#      - "8889:8889"   # Prometheus exporter metrics
#      - "13133:13133" # health_check extension
#      - "4317:4317"   # OTLP gRPC receiver
#      - "55679:55679" # zpages extension
#      - "12345:12345" # zpages extension
EOF

bash ${path}/start.sh

六、spark dependencies（与elasticsearch 集成）
6.1 生成cacerts证书，提供给dependencies使用

path=/data/jaeger
mkdir ${path} -p
cd ${path}
#进入容器
docker run --rm -it --entrypoint bash -u root  -v `pwd`/certs/:/certs/  jaegertracing/spark-dependencies 
#获取elasticsearch的https证书
openssl s_client -connect 192.168.11.100:9200 < /dev/null | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' >> ca.crt

#把elasticsearch的https证书导入cacerts
echo 'yes'|keytool -import -alias elasticsearch -keystore /certs/cacerts -storepass "123456" -file ca.crt  
#退出容器
exit

cat > ${path}/denpendencies.sh << 'EOF'
#!/bin/bash
cd `dirname $0`

docker run --rm -it \
-e STORAGE=elasticsearch \
-e ES_NODES=https://192.168.11.100:9200 \
-e ES_TIME_RANGE=24h \
-e ES_USERNAME=elastic \
-e ES_PASSWORD=xxxxxxxx \
-v `pwd`/certs/cacerts:/usr/local/openjdk-8/lib/security/cacerts \
-v /etc/localtime:/etc/localtime \
jaegertracing/spark-dependencies
EOF
bash ${path}/denpendencies.sh

demo1

path=/data/demo
mkdir ${path} -p
cd ${path}

cat >${path}/start.sh << 'EOF'
#!/bin/bash
docker rm -f demo-server
docker run  -d \
--restart=always \
--name demo-server \
-p 7080:7080 \
-e OTEL_EXPORTER_OTLP_ENDPOINT=192.168.11.100:4317 \
zengxiangbang/demo_demo-server

docker rm -f demo-client
docker run  -d \
--restart=always \
--name demo-client \
--net host \
-e OTEL_EXPORTER_OTLP_ENDPOINT=192.168.11.100:4317 \
-e DEMO_SERVER_ENDPOINT=http://192.168.11.100:7080/hello \
zengxiangbang/demo_demo-client
EOF
bash ${path}/start.sh

dem2

path=/data/hotrod
mkdir ${path} -p
cat >${path}/start.sh << 'EOF'
#!/bin/bash
docker rm -f hotrod

cd `dirname $0`
docker run -d \
--restart=always \
--name=hotrod \
-v /etc/localtime:/etc/localtime \
-p 8080:8080 \
-p 8081:8081 \
-p 8083:8083 \
-e OTEL_EXPORTER_JAEGER_ENDPOINT=http://192.168.11.100:14268/api/traces \
jaegertracing/example-hotrod:1.45 all -m prometheus
#-e OTEL_EXPORTER_JAEGER_AGENT_HOST=192.168.11.100 \
#-e OTEL_EXPORTER_JAEGER_AGENT_PORT=6831 \
EOF

bash ${path}/start.sh

#开放firewalld端口

firewall-cmd --permanent --add-port=8080/tcp
firewall-cmd --permanent --add-port=8081/tcp
firewall-cmd --permanent --add-port=8083/tcp
firewall-cmd --reload

#访问hotrod
http://192.168.11.100:8081/

demo3、自动生成

docker run --net host --rm yurishkuro/microsim -d 60s

---

spanmetricsconnectors otel-config.yaml 配置方法 ,未来偏向于用这种方法

cat >${path}/otel-config.yaml << 'EOF'
receivers:
  jaeger:
    protocols:
      thrift_http:
        endpoint: :14278

  otlp:
    protocols:
      grpc:
        endpoint: :4317
      http:
        endpoint: :4318

exporters:
  jaeger:
    endpoint: 192.168.11.100:14250
    tls:
      insecure: true

  prometheus:
    endpoint: "0.0.0.0:8889"

  prometheusremotewrite:
    endpoint: http://192.168.11.100:9090/api/v1/write
    target_info:
       enabled: true

connectors:
  spanmetrics:
    histogram:
      explicit:
        buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms]
    dimensions:
      - name: http.method
        default: GET
      - name: http.status_code
    dimensions_cache_size: 1000
    aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE"
    metrics_flush_interval: 15s

processors:
  batch:

extensions:
  health_check:
  pprof:
    endpoint: :1888
  zpages:
    endpoint: :55679

service:
  extensions: [pprof, zpages, health_check]
  pipelines:
    traces:
      receivers: [jaeger,otlp]
      exporters: [spanmetrics, jaeger]
    metrics:
      receivers: [spanmetrics]
      exporters: [prometheusremotewrite]
EOF

另一配置

receivers:
  jaeger:
    protocols:
      thrift_http:
        endpoint: :14268

  otlp:
    protocols:
      grpc:
        endpoint: :4317
      http:
        endpoint: :4318

exporters:
  jaeger:
    endpoint: 10.2.3.8:14250
    tls:
      insecure: true

  prometheus:
    endpoint: "10.2.3.8:9464"
    resource_to_telemetry_conversion:
      enabled: true
    enable_open_metrics: true

  prometheusremotewrite:
    endpoint: http://10.2.3.8:9090/api/v1/write
    target_info:
       enabled: true

connectors:
  spanmetrics:
    histogram:
      explicit:
        buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms]
    dimensions:
      - name: http.method
        default: GET
      - name: http.status_code
    dimensions_cache_size: 1000
    aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE"

processors:
  batch:
  metricstransform/insert:
    transforms:
      - include: calls
        match_type: strict
        action: insert
        new_name: calls_total
        operations:
        - action: update_label
          label: span.name
          new_label: operation
      - include: duration
        match_type: strict
        action: insert
        new_name: latency
        operations:
          - action: update_label
            label: span.name
            new_label: operation

extensions:
  health_check:
  pprof:
    endpoint: :1888
  zpages:
    endpoint: :55679

service:
  extensions: [pprof, zpages, health_check]
  pipelines:
    traces:
      receivers: [jaeger,otlp]
      processors: [batch]
      exporters: [spanmetrics,jaeger]

    metrics:
      receivers: [spanmetrics]
      processors: [metricstransform/insert]
      exporters: [prometheusremotewrite]