- loki日志系统介绍
- loki是主服务器，负责存储日志和处理查询
- promtail是专为loki定制的代理，负责收集日志并将其发送给 loki
- Grafana用于 UI展示
- 架构
- Distributor 收到 HTTP 请求，用于存储流数据
- 通过 hash 环对数据流进行 hash
- Distributor将数据流发送到对应的Ingester及其副本上
- Ingester 新建 Chunk 或将数据追加到已有Chunk 上
- Distributor通过 HTTP连接发送响应信息
- 组件介绍
- Distributor：Distributor 是客户端连接的组件，用于收集日志。Distributor会对接收到的日志流进行正确性校验，并将验证后的chunk日志块分批并行发送到Ingester。
- ingester： 接收来自Distributor的日志流，并将日志压缩后存放到所连接的存储后端，每个Ingester 的生命周期有ENDING, JOINING, ACTIVE, LEAVING 和 UNHEALTHY 五种状态。处于JOINING和ACTIVE状态的Ingester可以接受写请求，处于ACTIVE和LEAVING状态时可以接受读请求
- Querier：用来查询日志，可以直接从 Ingester 和后端存储中查询数据，Querier 就会查找索引来确定所有匹配 chunk ，然后对选中的日志进行 grep并返回查询结果，Querier可能会接收到重复的数据，所以其内置了去重的功能，对拥有同样时间戳、标签组和消息内容的日志进行去重处理
- Query Frontend：Query frontend会对请求做一些调整，并将请求放入一个内部的队列中。在该场景中，Querier作为workers 不断从队列中获取任务、执行任务，并将结果返回给Query frontend用于聚合。
- 写：日志数据的写主要依托的是Distributor和Ingester两个组件
- 读：主要用于Querier
- Querier 收到 HTTP 请求
- Querier 将请求发送至Ingester 用以获取内存数据
- Ingester 收到请求后返回符合条件的数据
- 如果没有Ingester 返回数据，Querier 从后端存储加载数据并执行查询
- Querier 遍历所有数据并进行去重处理，通过HTTP连接返回最终结果
- loki配置
apiVersion: v1 kind: ConfigMap metadata: annotations: meta.helm.sh/release-name: loki meta.helm.sh/release-namespace: loki name: loki namespace: loki data: config.yaml: | auth_enabled: false server: http_listen_port: 3100 distributor: ring: kvstore: store: memberlist memberlist: join_members: - loki-memberlist ingester: lifecycler: #配置ingester的生命周期，以及在哪里注册以进行发现 ring: kvstore: store: memberlist #用于ring的后端存储 replication_factor: 3 #写入和读取的ingester数量，至少为1（为了冗余和弹性，默认情况下为3） chunk_idle_period: 10m #如果块没有达到最大的块大小，那么刷新之前，块应该在内存中不更新多长时间 chunk_block_size: 262144 chunk_encoding: snappy #用于块的压缩算法gzip，lz4，snappy chunk_retain_period: 5m #块刷新后应该在内存中保留多长时间 max_transfer_retries: 0 #在离开之前尝试和传输块的次数。回退到刷新到存储 limits_config: enforce_metric_name: false #强制每个样本都有一个指标名称 reject_old_samples: true #拒绝旧样本 reject_old_samples_max_age: 144h #拒绝旧样本数最大时限 max_streams_matchers_per_query: 5000 #每个查询的流匹配器的最大数量 ingestion_burst_size_mb: 999 #修改每用户摄入速率限制，即每秒样本量，默认6m ingestion_rate_mb: 512 #修改每用户摄入速率限制，即每秒样本量，默认值4m schema_config: #配置从特定时间段开始应该使用哪些索引模式 configs: - from: 2020-09-07 #创建索引日期，如果这是唯一的schema_config, store: boltdb-shipper #用于索引的存储，aws, aws-dynamo, gcp, bigtable，boltdb-shipper，bigtable-hashed，cassandra, boltdb object_store: aws schema: v11 index: prefix: loki_index_ period: 24h #每张表的时间范围6天 storage_config: boltdb_shipper: #配置对象存储中的索引存储 shared_store: s3 #用于保存boltdb文件的共享存储 active_index_directory: /var/loki/index #索引文件存储地址 cache_location: /var/loki/cache #用于恢复 boltDB 文件以进行查询的缓存位置 cache_ttl: 24h aws: s3: "s3://UlcOP9wcI17umlHJrncK:c7G9Wc3d5M8BdoZBXSCZJ25qlFt1eaTyaIbfvOFr@minio.mcd-infra:9000/hetu-fr-loki" s3forcepathstyle: true #设置为 true 强制请求使用路径式寻址 chunk_store_config: max_look_back_period: 0s #最大可查询历史日期 0s table_manager: retention_deletes_enabled: true #用于表保留删除的主“开关” retention_period: 24h # 表的保留期1天 query_range: align_queries_with_step: true #改变传入的查询以使其开始和结束与他们的步骤保持一致 max_retries: 5 #最大重试次数 split_queries_by_interval: 10m #按照10m间隔查询 cache_results: true #缓存查询结果 results_cache: cache: enable_fifocache: true #开启内存缓存 fifocache: #先进先出缓存 max_size_items: 1024 #缓存中条目的最大数量 validity: 6h #缓存的有效期 frontend_worker: frontend_address: loki-query-frontend:9095 frontend: max_outstanding_per_tenant: 300 #每个前端每个租户的最大未完成请求数 log_queries_longer_than: 10s #记录比指定持续时间慢的查询 compress_responses: true compactor: shared_store: filesystem #用于存储boltdb文件的共享存储，gcs, s3, azure, swift, filesystem ruler: storage: type: local local: directory: /etc/loki/rules ring: kvstore: store: memberlist rule_path: /tmp/loki/scratch alertmanager_url: https://alertmanager-m8-x.mcd.megvii-inc.com external_url: https://alertmanager-m8-x.mcd.megvii-inc.comloki-gateway配置
apiVersion: v1 data: nginx.conf: | worker_processes 5; ## Default: 1 error_log /dev/stderr; pid /tmp/nginx.pid; worker_rlimit_nofile 8192; events { worker_connections 4096; ## Default: 1024 } http { client_body_temp_path /tmp/client_temp; proxy_temp_path /tmp/proxy_temp_path; fastcgi_temp_path /tmp/fastcgi_temp; uwsgi_temp_path /tmp/uwsgi_temp; scgi_temp_path /tmp/scgi_temp; default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] $status ' '"$request" $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log /dev/stderr main; sendfile on; tcp_nopush on; resolver kube-dns.kube-system.svc.cluster.local; server { listen 8080; location = / { return 200 'OK'; auth_basic off; } location = /api/prom/push { proxy_pass http://loki-distributor.loki.svc.cluster.local:3100$request_uri; } location = /api/prom/tail { proxy_pass http://loki-querier.loki.svc.cluster.local:3100$request_uri; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; } location ~ /api/prom/.* { proxy_pass http://loki-query-frontend.loki.svc.cluster.local:3100$request_uri; } location = /loki/api/v1/push { proxy_pass http://loki-distributor.loki.svc.cluster.local:3100$request_uri; } location = /loki/api/v1/tail { proxy_pass http://loki-querier.loki.svc.cluster.local:3100$request_uri; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; } location ~ /loki/api/.* { proxy_pass http://loki-query-frontend.loki.svc.cluster.local:3100$request_uri; } } } kind: ConfigMap metadata: annotations: meta.helm.sh/release-name: loki meta.helm.sh/release-namespace: loki name: loki-gateway namespace: loki -
promtail配置文件
apiVersion: v1 kind: ConfigMap metadata: annotations: meta.helm.sh/release-name: promtail meta.helm.sh/release-namespace: loki name: promtail namespace: loki data: promtail.yaml: | client: backoff_config: max_period: 5m #重试之间的最大回退时间 max_retries: 10 #重试的最大次数 min_period: 500ms #重试之间的初始回退时间 batchsize: 1048576 #发送批处理之前要积累的最大批处理量（以字节为单位） batchwait: 3s #发送一批日志前的最大等待时间，即使该批次日志数据未满 external_labels: {} timeout: 10s positions: #表示它已经读到了文件什么程度。当 Promtail 重新启动时需要它，以允许它从中断的地方继续读取日志 filename: /run/promtail/positions.yaml server: http_listen_port: 3101 target_config: sync_period: 5s #更新 positions 文件的周期 scrape_configs: - job_name: kubernetes-pods-name pipeline_stages: - docker: {} - multiline: firstline: ^\S+ kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: - __meta_kubernetes_pod_label_name target_label: __service__ - source_labels: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop regex: '' source_labels: - __service__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / source_labels: - __meta_kubernetes_namespace - __service__ target_label: job - action: replace source_labels: - __meta_kubernetes_namespace target_label: namespace - action: replace source_labels: - __meta_kubernetes_pod_name target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - job_name: kubernetes-pods-app pipeline_stages: - docker: {} - multiline: firstline: ^\S+ kubernetes_sd_configs: - role: pod relabel_configs: - action: drop regex: .+ source_labels: - __meta_kubernetes_pod_label_name - source_labels: - __meta_kubernetes_pod_label_app target_label: __service__ - source_labels: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop regex: '' source_labels: - __service__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / source_labels: - __meta_kubernetes_namespace - __service__ target_label: job - action: replace source_labels: - __meta_kubernetes_namespace target_label: namespace - action: replace source_labels: - __meta_kubernetes_pod_name target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - job_name: kubernetes-pods-direct-controllers pipeline_stages: - docker: {} - multiline: firstline: ^\S+ kubernetes_sd_configs: - role: pod relabel_configs: - action: drop regex: .+ separator: '' source_labels: - __meta_kubernetes_pod_label_name - __meta_kubernetes_pod_label_app - action: drop regex: '[0-9a-z-.]+-[0-9a-f]{8,10}' source_labels: - __meta_kubernetes_pod_controller_name - source_labels: - __meta_kubernetes_pod_controller_name target_label: __service__ - source_labels: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop regex: '' source_labels: - __service__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / source_labels: - __meta_kubernetes_namespace - __service__ target_label: job - action: replace source_labels: - __meta_kubernetes_namespace target_label: namespace - action: replace source_labels: - __meta_kubernetes_pod_name target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - job_name: kubernetes-pods-indirect-controller pipeline_stages: - docker: {} - multiline: firstline: ^\S+ kubernetes_sd_configs: - role: pod relabel_configs: - action: drop regex: .+ separator: '' source_labels: - __meta_kubernetes_pod_label_name - __meta_kubernetes_pod_label_app - action: keep regex: '[0-9a-z-.]+-[0-9a-f]{8,10}' source_labels: - __meta_kubernetes_pod_controller_name - action: replace regex: '([0-9a-z-.]+)-[0-9a-f]{8,10}' source_labels: - __meta_kubernetes_pod_controller_name target_label: __service__ - source_labels: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop regex: '' source_labels: - __service__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / source_labels: - __meta_kubernetes_namespace - __service__ target_label: job - action: replace source_labels: - __meta_kubernetes_namespace target_label: namespace - action: replace source_labels: - __meta_kubernetes_pod_name target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_uid - __meta_kubernetes_pod_container_name target_label: __path__ - job_name: kubernetes-pods-static pipeline_stages: - docker: {} - multiline: firstline: ^\S+ kubernetes_sd_configs: - role: pod relabel_configs: - action: drop regex: '' source_labels: - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror - action: replace source_labels: - __meta_kubernetes_pod_label_component target_label: __service__ - source_labels: - __meta_kubernetes_pod_node_name target_label: __host__ - action: drop regex: '' source_labels: - __service__ - action: labelmap regex: __meta_kubernetes_pod_label_(.+) - action: replace replacement: $1 separator: / source_labels: - __meta_kubernetes_namespace - __service__ target_label: job - action: replace source_labels: - __meta_kubernetes_namespace target_label: namespace - action: replace source_labels: - __meta_kubernetes_pod_name target_label: pod - action: replace source_labels: - __meta_kubernetes_pod_container_name target_label: container - replacement: /var/log/pods/*$1/*.log separator: / source_labels: - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror - __meta_kubernetes_pod_container_name target_label: __path__
07-12
5060
5060
“相关推荐”对你有帮助么？
-
非常没帮助 -
没帮助 -
一般 -
有帮助 -
非常有帮助
提交
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
