在skywalking-oap的pod日志中抛出了以下异常
2024-05-05 02:20:01,464 io.kubernetes.client.informer.cache.ReflectorRunnable 84 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] INFO [] - class io.kubernetes.client.openapi.models.V1Pod#Start listing and watching...
2024-05-05 02:20:01,468 io.kubernetes.client.informer.cache.ReflectorRunnable 282 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] ERROR [] - [9.0.0-89644fe (20240217220813)] class io.kubernetes.client.openapi.models.V1Pod#Reflector loop failed unexpectedly
io.kubernetes.client.openapi.ApiException:
at io.kubernetes.client.openapi.ApiClient.handleResponse(ApiClient.java:973) ~[client-java-api-14.0.0.jar:?]
at io.kubernetes.client.openapi.ApiClient.execute(ApiClient.java:885) ~[client-java-api-14.0.0.jar:?]
at io.kubernetes.client.informer.SharedInformerFactory$1.list(SharedInformerFactory.java:207) ~[client-java-14.0.0.jar:?]
at io.kubernetes.client.informer.cache.ReflectorRunnable.run(ReflectorRunnable.java:88) ~[client-java-14.0.0.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) [?:?]
at java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
at java.lang.Thread.run(Unknown Source) [?:?]
2024-05-05 02:20:02,468 io.kubernetes.client.informer.cache.ReflectorRunnable 84 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] INFO [] - class io.kubernetes.client.openapi.models.V1Pod#Start listing and watching...
2024-05-05 02:20:02,472 io.kubernetes.client.informer.cache.ReflectorRunnable 282 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] ERROR [] - [9.0.0-89644fe (20240217220813)] class io.kubernetes.client.openapi.models.V1Pod#Reflector loop failed unexpectedly
io.kubernetes.client.openapi.ApiException:
at io.kubernetes.client.openapi.ApiClient.handleResponse(ApiClient.java:973) ~[client-java-api-14.0.0.jar:?]
at io.kubernetes.client.openapi.ApiClient.execute(ApiClient.java:885) ~[client-java-api-14.0.0.jar:?]
at io.kubernetes.client.informer.SharedInformerFactory$1.list(SharedInformerFactory.java:207) ~[client-java-14.0.0.jar:?]
at io.kubernetes.client.informer.cache.ReflectorRunnable.run(ReflectorRunnable.java:88) ~[client-java-14.0.0.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) [?:?]
at java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
at java.lang.Thread.run(Unknown Source) [?:?]
2024-05-05 02:20:03,473 io.kubernetes.client.informer.cache.ReflectorRunnable 84 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] INFO [] - class io.kubernetes.client.openapi.models.V1Pod#Start listing and watching...
2024-05-05 02:20:03,477 io.kubernetes.client.informer.cache.ReflectorRunnable 282 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] ERROR [] - [9.0.0-89644fe (20240217220813)] class io.kubernetes.client.openapi.models.V1Pod#Reflector loop failed unexpectedly
io.kubernetes.client.openapi.ApiException:
at io.kubernetes.client.openapi.ApiClient.handleResponse(ApiClient.java:973) ~[client-java-api-14.0.0.jar:?]
at io.kubernetes.client.openapi.ApiClient.execute(ApiClient.java:885) ~[client-java-api-14.0.0.jar:?]
at io.kubernetes.client.informer.SharedInformerFactory$1.list(SharedInformerFactory.java:207) ~[client-java-14.0.0.jar:?]
at io.kubernetes.client.informer.cache.ReflectorRunnable.run(ReflectorRunnable.java:88) ~[client-java-14.0.0.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) [?:?]
at java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
at java.lang.Thread.run(Unknown Source) [?:?]
2024-05-05 02:20:04,477 io.kubernetes.client.informer.cache.ReflectorRunnable 84 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] INFO [] - class io.kubernetes.client.openapi.models.V1Pod#Start listing and watching...
2024-05-05 02:20:04,481 io.kubernetes.client.informer.cache.ReflectorRunnable 282 [controller-reflector-io.kubernetes.client.openapi.models.V1Pod-1] ERROR [] - [9.0.0-89644fe (20240217220813)] class io.kubernetes.client.openapi.models.V1Pod#Reflector loop failed unexpectedly
io.kubernetes.client.openapi.ApiException:
at io.kubernetes.client.openapi.ApiClient.handleResponse(ApiClient.java:973) ~[client-java-api-14.0.0.jar:?]
at io.kubernetes.client.openapi.ApiClient.execute(ApiClient.java:885) ~[client-java-api-14.0.0.jar:?]
at io.kubernetes.client.informer.SharedInformerFactory$1.list(SharedInformerFactory.java:207) ~[client-java-14.0.0.jar:?]
at io.kubernetes.client.informer.cache.ReflectorRunnable.run(ReflectorRunnable.java:88) ~[client-java-14.0.0.jar:?]
at java.util.concurrent.Executors$RunnableAdapter.call(Unknown Source) [?:?]
at java.util.concurrent.FutureTask.runAndReset(Unknown Source) [?:?]
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source) [?:?]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source) [?:?]
at java.lang.Thread.run(Unknown Source) [?:?]
保存老的serviceaccount
kubectl get ServiceAccount -n ops skywalking-oap -o yaml >serviceaccount-old.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
annotations:
meta.helm.sh/release-name: skywalking
meta.helm.sh/release-namespace: ops
creationTimestamp: "2023-08-03T02:54:51Z"
labels:
app: skywalking
app.kubernetes.io/managed-by: Helm
chart: skywalking-4.3.0
component: oap
heritage: Helm
release: skywalking
name: skywalking-oap
namespace: ops
resourceVersion: "77332520"
selfLink: /api/v1/namespaces/ops/serviceaccounts/skywalking-oap
uid: 5c64e3b5-3acd-4f43-a03f-b9000e3703f4
secrets:
- name: skywalking-oap-token-lv6xc
可以查询到secrets都是正常的
# kubectl get secret -n ops|grep skywalking
prod-skywalking-oap-token-xsqvj kubernetes.io/service-account-token 3 603d
sh.helm.release.v1.skywalking.v1 helm.sh/release.v1 1 378d
skywalking-oap-token-lv6xc kubernetes.io/service-account-token 3 378d
# kubectl describe secret -n ops skywalking-oap-token-lv6xc
Name: skywalking-oap-token-lv6xc
Namespace: ops
Labels: <none>
Annotations: kubernetes.io/service-account.name: skywalking-oap
kubernetes.io/service-account.uid: 5c64e3b5-3acd-4f43-a03f-b9000e3703f4
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1350 bytes
namespace: 3 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6InE5ZE1pc2tOTVZoN1Jjam96OGZVNTV4YzFFX2Mzc1ZQMXRuUERhQ0JLSFUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJvcHMiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlY3JldC5uYW1lIjoic2t5d2Fsa2luZy1vYXAtdG9rZW4tbHY2eGMiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoic2t5d2Fsa2luZy1vYXAiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI1YzY0ZTNiNS0zYWNkLTRmNDMtYTAzZi1iOTAwMGUzNzAzZjQiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6b3BzOnNreXdhbGtpbmctb2FwIn0.CzFP_WStl7UsLg8Klc8zV6EPw_SBjLh-vcjYp-zkXQIubo1GYnwZqRVrKIDUShuLheo-Vrh9-VABWtDDJgn6vtdvMqTSxLqYmOXEO_plmPMyJcExKAXynBg1XRLv3oWaE2e3xlTH2jMvyfG9R0eRWPlXfAvrkML3bfjyQhdhRzXmX0FKWSuCDSRJH3iwasT4KyD1MLnAVtqkoKc3uCmEjonX4HrLjMI_8n6JvjgdYaeqsQnwY2Jxs21KPdJMJQl1Cx2ycOBRiWyppzt6fnmtldJBCogXEMG1GCSWjfwt9umMUt9llDeAwZs598sOpCKEmV-JqpkFc0GpjeXcztya6g
出现这个问题的原因是我复制了部分helm 8版本的权限yaml文件,并自定义了yaml配置文件
终于解决了问题,使用了更大的ServiceAccount权限进行配置,大致如下
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
app: skywalking
name: skywalking-oap
namespace: ops
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: skywalking
namespace: ops
labels:
app: skywalking
rules:
- apiGroups: [""]
resources: ["pods", "endpoints", "services", "nodes"]
verbs: ["get", "watch", "list"]
- apiGroups: ["extensions"]
resources: ["deployments", "replicasets"]
verbs: ["get", "watch", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: skywalking
namespace: ops
labels:
app: skywalking
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: skywalking
subjects:
- kind: ServiceAccount
name: skywalking-oap
namespace: ops