prometheus WAL log samples: log series: write /data/wal/000025: file already closed

k8s helm 部署 prometheus，各个监控都处于 down 的状态

[root@k8s-master-01 ~]# kubectl exec  -it prometheus-prometheus-server-697cccff9c-qtrf7 -c prometheus-server  sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/prometheus $ cd /data/
/data $ ls -l
total 1
drwxr-sr-x    3 nobody   nogroup         68 Dec 21 19:00 01GMV0XA1DFQWDT1FJ07QEM55J
drwxr-sr-x    3 nobody   nogroup         68 Dec 22 13:00 01GMWYPT7WXCFBSRPYYDXS0K2X
drwxr-sr-x    3 nobody   nogroup         68 Dec 23 07:00 01GMYWGDDA8EG7T8K7FP9C7WD0
drwxr-sr-x    3 nobody   nogroup         68 Dec 23 13:00 01GMZH3GRQ392621WMW9WJVCZP
drwxr-sr-x    3 nobody   nogroup         68 Dec 23 15:00 01GMZQZ1KN5PTFYQ9W4REEXB3N
drwxr-sr-x    3 nobody   nogroup         68 Dec 23 17:00 01GMZYTRVP65M0RP7VB9MCZAAY
-rw-------    1 nobody   nogroup          2 Dec 21 00:51 lock
drwxr-sr-x    2 nobody   nogroup         76 Dec 23 21:51 wal
/data $ du -h
822.7M  ./wal
217.6M  ./01GMV0XA1DFQWDT1FJ07QEM55J/chunks
247.4M  ./01GMV0XA1DFQWDT1FJ07QEM55J
228.3M  ./01GMWYPT7WXCFBSRPYYDXS0K2X/chunks
258.1M  ./01GMWYPT7WXCFBSRPYYDXS0K2X
226.4M  ./01GMYWGDDA8EG7T8K7FP9C7WD0/chunks
256.1M  ./01GMYWGDDA8EG7T8K7FP9C7WD0
74.5M   ./01GMZH3GRQ392621WMW9WJVCZP/chunks
97.3M   ./01GMZH3GRQ392621WMW9WJVCZP
25.0M   ./01GMZQZ1KN5PTFYQ9W4REEXB3N/chunks
45.5M   ./01GMZQZ1KN5PTFYQ9W4REEXB3N
25.0M   ./01GMZYTRVP65M0RP7VB9MCZAAY/chunks
45.5M   ./01GMZYTRVP65M0RP7VB9MCZAAY
1.7G  

...

/data $ df -h
Filesystem                Size      Used Available Use% Mounted on
overlay                  45.1G     12.3G     32.8G  27% /
tmpfs                    64.0M         0     64.0M   0% /dev
tmpfs                     7.7G         0      7.7G   0% /sys/fs/cgroup
/dev/mapper/centos-root
                         45.1G     12.3G     32.8G  27% /prometheus
10.0.2.46:vol_f1ab92b622d91c630a70a0c7a90ba91c
                          2.0G      1.8G    231.6M  89% /data
/dev/mapper/centos-root
                         45.1G     12.3G     32.8G  27% /dev/termination-log
/dev/mapper/centos-root
                         45.1G     12.3G     32.8G  27% /etc/config
/dev/mapper/centos-root
                         45.1G     12.3G     32.8G  27% /etc/resolv.conf
/dev/mapper/centos-root
                         45.1G     12.3G     32.8G  27% /etc/hostname
/dev/mapper/centos-root
                         45.1G     12.3G     32.8G  27% /etc/hosts
shm                      64.0M         0     64.0M   0% /dev/shm
tmpfs                    15.4G     12.0K     15.4G   0% /var/run/secrets/kubernetes.io/serviceaccount
tmpfs                     7.7G         0      7.7G   0% /proc/acpi
tmpfs                    64.0M         0     64.0M   0% /proc/kcore
tmpfs                    64.0M         0     64.0M   0% /proc/keys
tmpfs                    64.0M         0     64.0M   0% /proc/timer_list
tmpfs                    64.0M         0     64.0M   0% /proc/timer_stats
tmpfs                    64.0M         0     64.0M   0% /proc/sched_debug
tmpfs                     7.7G         0      7.7G   0% /proc/scsi
tmpfs                     7.7G         0      7.7G   0% /sys/firmware

数据保留默认15天，修改数据保留时长

retention 数据留存时间单位
https://access.redhat.com/documentation/zh-cn/openshift_container_platform/4.11/html/monitoring/modifying-retention-time-and-size-for-prometheus-metrics-data_configuring-the-monitoring-stack

配置 retention 参数无效

https://github.com/prometheus/prometheus/issues/8005

切换到 prometheus 2.22.1 版本，--storage.tsdb.retention.time 、--storage.tsdb.retention.size 生效