skywalking-oap抛出kubernetes.client.openapi.ApiException: .....java.lang.Thread.run(Unknown Source)

在skywalking-oap的pod日志中抛出了以下异常

2024-05-05 02:20:01,464 io.kubernetes.client.informer.cache.ReflectorRunnable 84 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] INFO  [] - class io.kubernetes.client.openapi.models.V1Pod#Start listing and watching...
2024-05-05 02:20:01,468 io.kubernetes.client.informer.cache.ReflectorRunnable 282 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] ERROR [] - [9.0.0-89644fe (20240217220813)] class io.kubernetes.client.openapi.models.V1Pod#Reflector loop failed unexpectedly
io.kubernetes.client.openapi.ApiException:
        at io.kubernetes.client.openapi.ApiClient.handleResponse(ApiClient.java:973) ~[client-java-api-14.0.0.jar:?]
        at io.kubernetes.client.openapi.ApiClient.execute(ApiClient.java:885) ~[client-java-api-14.0.0.jar:?]
        at io.kubernetes.client.informer.SharedInformerFactory$1.list(SharedInformerFactory.java:207) ~[client-java-14.0.0.jar:?]
        at io.kubernetes.client.informer.cache.ReflectorRunnable.run(ReflectorRunnable.java:88) ~[client-java-14.0.0.jar:?]
        at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) [?:?]
        at java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) [?:?]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
        at java.lang.Thread.run(Unknown Source) [?:?]
2024-05-05 02:20:02,468 io.kubernetes.client.informer.cache.ReflectorRunnable 84 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] INFO  [] - class io.kubernetes.client.openapi.models.V1Pod#Start listing and watching...
2024-05-05 02:20:02,472 io.kubernetes.client.informer.cache.ReflectorRunnable 282 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] ERROR [] - [9.0.0-89644fe (20240217220813)] class io.kubernetes.client.openapi.models.V1Pod#Reflector loop failed unexpectedly
io.kubernetes.client.openapi.ApiException:
        at io.kubernetes.client.openapi.ApiClient.handleResponse(ApiClient.java:973) ~[client-java-api-14.0.0.jar:?]
        at io.kubernetes.client.openapi.ApiClient.execute(ApiClient.java:885) ~[client-java-api-14.0.0.jar:?]
        at io.kubernetes.client.informer.SharedInformerFactory$1.list(SharedInformerFactory.java:207) ~[client-java-14.0.0.jar:?]
        at io.kubernetes.client.informer.cache.ReflectorRunnable.run(ReflectorRunnable.java:88) ~[client-java-14.0.0.jar:?]
        at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) [?:?]
        at java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) [?:?]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
        at java.lang.Thread.run(Unknown Source) [?:?]
2024-05-05 02:20:03,473 io.kubernetes.client.informer.cache.ReflectorRunnable 84 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] INFO  [] - class io.kubernetes.client.openapi.models.V1Pod#Start listing and watching...
2024-05-05 02:20:03,477 io.kubernetes.client.informer.cache.ReflectorRunnable 282 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] ERROR [] - [9.0.0-89644fe (20240217220813)] class io.kubernetes.client.openapi.models.V1Pod#Reflector loop failed unexpectedly
io.kubernetes.client.openapi.ApiException:
        at io.kubernetes.client.openapi.ApiClient.handleResponse(ApiClient.java:973) ~[client-java-api-14.0.0.jar:?]
        at io.kubernetes.client.openapi.ApiClient.execute(ApiClient.java:885) ~[client-java-api-14.0.0.jar:?]
        at io.kubernetes.client.informer.SharedInformerFactory$1.list(SharedInformerFactory.java:207) ~[client-java-14.0.0.jar:?]
        at io.kubernetes.client.informer.cache.ReflectorRunnable.run(ReflectorRunnable.java:88) ~[client-java-14.0.0.jar:?]
        at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) [?:?]
        at java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) [?:?]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
        at java.lang.Thread.run(Unknown Source) [?:?]
2024-05-05 02:20:04,477 io.kubernetes.client.informer.cache.ReflectorRunnable 84 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] INFO  [] - class io.kubernetes.client.openapi.models.V1Pod#Start listing and watching...
2024-05-05 02:20:04,481 io.kubernetes.client.informer.cache.ReflectorRunnable 282 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] ERROR [] - [9.0.0-89644fe (20240217220813)] class io.kubernetes.client.openapi.models.V1Pod#Reflector loop failed unexpectedly
io.kubernetes.client.openapi.ApiException:
        at io.kubernetes.client.openapi.ApiClient.handleResponse(ApiClient.java:973) ~[client-java-api-14.0.0.jar:?]
        at io.kubernetes.client.openapi.ApiClient.execute(ApiClient.java:885) ~[client-java-api-14.0.0.jar:?]
        at io.kubernetes.client.informer.SharedInformerFactory$1.list(SharedInformerFactory.java:207) ~[client-java-14.0.0.jar:?]
        at io.kubernetes.client.informer.cache.ReflectorRunnable.run(ReflectorRunnable.java:88) ~[client-java-14.0.0.jar:?]
        at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) [?:?]
        at java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) [?:?]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
        at java.lang.Thread.run(Unknown Source) [?:?]

保存老的serviceaccount
kubectl get ServiceAccount -n ops skywalking-oap -o yaml >serviceaccount-old.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  annotations:
    meta.helm.sh/release-name: skywalking
    meta.helm.sh/release-namespace: ops
  creationTimestamp: "2023-08-03T02:54:51Z"
  labels:
    app: skywalking
    app.kubernetes.io/managed-by: Helm
    chart: skywalking-4.3.0
    component: oap
    heritage: Helm
    release: skywalking
  name: skywalking-oap
  namespace: ops
  resourceVersion: "77332520"
  selfLink: /api/v1/namespaces/ops/serviceaccounts/skywalking-oap
  uid: 5c64e3b5-3acd-4f43-a03f-b9000e3703f4
secrets:
- name: skywalking-oap-token-lv6xc

可以查询到secrets都是正常的

#  kubectl get  secret -n ops|grep skywalking
prod-skywalking-oap-token-xsqvj       kubernetes.io/service-account-token   3      603d
sh.helm.release.v1.skywalking.v1      helm.sh/release.v1                    1      378d
skywalking-oap-token-lv6xc            kubernetes.io/service-account-token   3      378d
#  kubectl describe  secret -n ops skywalking-oap-token-lv6xc
Name:         skywalking-oap-token-lv6xc
Namespace:    ops
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: skywalking-oap
              kubernetes.io/service-account.uid: 5c64e3b5-3acd-4f43-a03f-b9000e3703f4

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1350 bytes
namespace:  3 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6InE5ZE1pc2tOTVZoN1Jjam96OGZVNTV4YzFFX2Mzc1ZQMXRuUERhQ0JLSFUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJvcHMiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoic2t5d2Fsa2luZy1vYXAtdG9rZW4tbHY2eGMiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoic2t5d2Fsa2luZy1vYXAiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI1YzY0ZTNiNS0zYWNkLTRmNDMtYTAzZi1iOTAwMGUzNzAzZjQiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6b3BzOnNreXdhbGtpbmctb2FwIn0.CzFP_WStl7UsLg8Klc8zV6EPw_SBjLh-vcjYp-zkXQIubo1GYnwZqRVrKIDUShuLheo-Vrh9-VABWtDDJgn6vtdvMqTSxLqYmOXEO_plmPMyJcExKAXynBg1XRLv3oWaE2e3xlTH2jMvyfG9R0eRWPlXfAvrkML3bfjyQhdhRzXmX0FKWSuCDSRJH3iwasT4KyD1MLnAVtqkoKc3uCmEjonX4HrLjMI_8n6JvjgdYaeqsQnwY2Jxs21KPdJMJQl1Cx2ycOBRiWyppzt6fnmtldJBCogXEMG1GCSWjfwt9umMUt9llDeAwZs598sOpCKEmV-JqpkFc0GpjeXcztya6g

出现这个问题的原因是我复制了部分helm 8版本的权限yaml文件，并自定义了yaml配置文件
终于解决了问题，使用了更大的ServiceAccount权限进行配置，大致如下

apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: skywalking
  name: skywalking-oap
  namespace: ops
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: skywalking
  namespace: ops
  labels:
    app: skywalking
rules:
  - apiGroups: [""]
    resources: ["pods", "endpoints", "services", "nodes"]
    verbs: ["get", "watch", "list"]
  - apiGroups: ["extensions"]
    resources: ["deployments", "replicasets"]
    verbs: ["get", "watch", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: skywalking
  namespace: ops
  labels:
    app: skywalking
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: skywalking
subjects:
  - kind: ServiceAccount
    name: skywalking-oap
    namespace: ops