1背景:
kubeadm join集群时,出现JWS token找不到的错误
2现象
通过token执行如下代码时,会出现 “error execution phase preflight: couldn’t validate the identity of the API Server: could not find a JWS signature in the cluster-info ConfigMap for token ID” 这样的报错,本文主要分析一下报错原因
3分析过程
这个问题在kubeadm代码中 discovery/token/token.go 中的getClusterInfo函数上报,该函数代码如下
在这里插入代码片// getClusterInfo creates a client from the given kubeconfig if the given client is nil,
// and requests the cluster info ConfigMap using PollImmediate.
// If a client is provided it will be used instead.
func getClusterInfo(client clientset.Interface, kubeconfig *clientcmdapi.Config, token *bootstraptokenv1.BootstrapTokenString, interval, duration time.Duration) (*v1.ConfigMap, error) {
var cm *v1.ConfigMap
var err error
// Create client from kubeconfig
if client == nil {
client, err = kubeconfigutil.ToClientSet(kubeconfig)
if err != nil {
return nil, err
}
}
klog.V(1).Infof("[discovery] Waiting for the cluster-info ConfigMap to receive a JWS signature"+
"for token ID %q", token.ID)
var lastError error
err = wait.PollUntilContextTimeout(context.Background(),
interval, duration, true,
func(ctx context.Context) (bool, error) {
cm, err = client.CoreV1().ConfigMaps(metav1.NamespacePublic).
Get(ctx, bootstrapapi.ConfigMapClusterInfo, metav1.GetOptions{})
if err != nil {
lastError = errors.Wrapf(err, "failed to request the cluster-info ConfigMap")
klog.V(1).Infof("[discovery] Retrying due to error: %v", lastError)
return false, nil
}
// Even if the ConfigMap is available the JWS signature is patched-in a bit later.
if _, ok := cm.Data[bootstrapapi.JWSSignatureKeyPrefix+token.ID]; !ok {
lastError = errors.Errorf("could not find a JWS signature in the cluster-info ConfigMap"+
" for token ID %q", token.ID)
klog.V(1).Infof("[discovery] Retrying due to error: %v", lastError)
return false, nil
}
return true, nil
})
if err != nil {
return nil, lastError
}
return cm, nil
由代码可知,其是查找cluster-info中Data中是否含有对应的token,没有的话就报错。那我们看一下函数调用流程
runControlPlanePrepareControlPlaneSubphase cmd/phases/join/controlplaneprepare.go
InitCfg cmd/join.go
TLSBootstrapCfg cmd/join.go
For discovery/discovery.go
DiscoverValidatedKubeConfig discovery/discovery.go
RetrieveValidatedConfigInfo discovery/token/token.go
retrieveValidatedConfigInfo discovery/token/token.go
getClusterInfo discovery/token/token.go
这里我们就需要考虑一下,我们创建的token会注入到cluster-info中吗?
kubeadm中token的创建有2种情况,一种是kubeadm init是创建,另一种直接通过kubeadm token create指令创建。我们先看一下kubeadm token create
在这里插入代码片// RunCreateToken generates a new bootstrap token and stores it as a secret on the server.
233 func RunCreateToken(out io.Writer, client clientset.Interface, cfgPath string, initCfg *kubeadmapiv1.InitConfiguration, printJoinCommand bool, certificateKey string, kubeConfigFile string) error {
234 // ClusterConfiguration is needed just for the call to LoadOrDefaultInitConfiguration
235 clusterCfg := &kubeadmapiv1.ClusterConfiguration{
236 // KubernetesVersion is not used, but we set this explicitly to avoid
237 // the lookup of the version from the internet when executing LoadOrDefaultInitConfiguration
238 KubernetesVersion: kubeadmconstants.CurrentKubernetesVersion.String(),
239 }
240 kubeadmscheme.Scheme.Default(clusterCfg)
241
242 // This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
243 klog.V(1).Infoln("[token] loading configurations")
244
245 internalcfg, err := configutil.LoadOrDefaultInitConfiguration(cfgPath, initCfg, clusterCfg, configutil.LoadOrDefaultConfigurationOptions{
246 SkipCRIDetect: true,
247 })
248 if err != nil {
249 return err
250 }
251
252 klog.V(1).Infoln("[token] creating token")
253 // 创建token
254 if err := tokenphase.CreateNewTokens(client, internalcfg.BootstrapTokens); err != nil {
255 return err
256 }
257
258 // if --print-join-command was specified, print a machine-readable full `kubeadm join` command
259 // otherwise, just print the token
260 if printJoinCommand {
261 skipTokenPrint := false
262 if certificateKey != "" {
// 是否打印token信息
263 ............
285 }
286
287 return nil
288 }
从上述代码可知,通过tokenphase.CreateNewTokens创建token, 代码路径 kubeadm/app/phases/bootstraptoken/node
35 // CreateNewTokens tries to create a token and fails if one with the same ID already exists
36 func CreateNewTokens(client clientset.Interface, tokens []bootstraptokenv1.BootstrapToken) error {
37 return UpdateOrCreateTokens(client, true, tokens)
38 }
39
40 // UpdateOrCreateTokens attempts to update a token with the given ID, or create if it does not already exist.
41 func UpdateOrCreateTokens(client clientset.Interface, failIfExists bool, tokens []bootstraptokenv1.BootstrapToken) error {
42
43 for _, token := range tokens {
44
45 secretName := bootstraputil.BootstrapTokenSecretName(token.Token.ID)
// 查询集群中secret是否包含该token信息
46 secret, err := client.CoreV1().Secrets(metav1.NamespaceSystem).Get(context.TODO(), secretName, metav1.GetOptions{})
47 if secret != nil && err == nil && failIfExists {
48 return errors.Errorf("a token with id %q already exists", token.Token.ID)
49 }
50 // 将token构建成secret格式
51 updatedOrNewSecret := bootstraptokenv1.BootstrapTokenToSecret(&token)
52
53 var lastError error
54 err = wait.PollUntilContextTimeout(
55 context.Background(),
56 kubeadmconstants.KubernetesAPICallRetryInterval,
57 kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration,
58 true, func(_ context.Context) (bool, error) {
59 if err := apiclient.CreateOrUpdateSecret(client, updatedOrNewSecret); err != nil {
60 lastError = errors.Wrapf(err, "failed to create or update bootstrap token with name %s", secretName)
61 return false, nil
62 }
63 return true, nil
64 })
65 if err != nil {
66 return lastError
67 }
68 }
69 return nil
70 }
主要关注的实现行是apiclient.CreateOrUpdateSecret(client, updateOrNewSecret)
在这里插入代码片140 // CreateOrUpdateSecret creates a Secret if the target resource doesn't exist. If the resource exists already, this function will update the resource instead.
141 func CreateOrUpdateSecret(client clientset.Interface, secret *v1.Secret) error {
142 var lastError error
143 err := wait.PollUntilContextTimeout(context.Background(),
144 apiCallRetryInterval, kubeadmapi.GetActiveTimeouts().KubernetesAPICall.Duration,
145 true, func(_ context.Context) (bool, error) {
146 ctx := context.Background()
147 if _, err := client.CoreV1().Secrets(secret.ObjectMeta.Namespace).Create(ctx, secret, metav1.CreateOptions{}); err != nil {
148 if !apierrors.IsAlreadyExists(err) {
149 lastError = errors.Wrap(err, "unable to create Secret")
150 return false, nil
151 }
152 if _, err := client.CoreV1().Secrets(secret.ObjectMeta.Namespace).Update(ctx, secret, metav1.UpdateOptions{}); err != nil {
153 lastError = errors.Wrap(err, "unable to update Secret")
154 return false, nil
155 }
156 }
157 return true, nil
158 })
159 if err == nil {
160 return nil
161 }
162 return lastError
163 }
注释说的很明确:创建或者更新集群中该secret,我们执行一次操作看一下效果,通过查看kubeadm token create执行流程,基本只管看到其对secrets进行了post创建
这里留下最后一个疑问:是谁将该secret同步到cluster-info configmap中, 答案是kube-controller-manager
controller/bootstrap/bootstrapsigner.go
在这里插入代码片271 func (e *Signer) listSecrets() []*v1.Secret {
272 secrets, err := e.secretLister.Secrets(e.secretNamespace).List(labels.Everything())
273 if err != nil {
274 utilruntime.HandleError(err)
275 return nil
276 }
277
278 items := []*v1.Secret{}
279 for _, secret := range secrets {
280 // 筛选是否是bootstrap token
281 if secret.Type == bootstrapapi.SecretTypeBootstrapToken {
282 items = append(items, secret)
283 }
284 }
285 return items
286 }
288 // getTokens returns a map of tokenID->tokenSecret. It ensures the token is
289 // valid for signing.
290 func (e *Signer) getTokens(ctx context.Context) map[string]string {
291 ret := map[string]string{}
292 secretObjs := e.listSecrets()
293 for _, secret := range secretObjs {
294 tokenID, tokenSecret, ok := validateSecretForSigning(ctx, secret)
295 if !ok {
296 continue
297 }
298
299 // Check and warn for duplicate secrets. Behavior here will be undefined.
300 if _, ok := ret[tokenID]; ok {
301 // This should never happen as we ensure a consistent secret name.
302 // But leave this in here just in case.
303 klog.FromContext(ctx).V(1).Info("Duplicate bootstrap tokens found for id, ignoring on the duplicate secret", "tokenID", tokenID, "ignoredSecret", klog.KObj(secret))
304 continue
305 }
306
307 // This secret looks good, add it to the list.
308 ret[tokenID] = tokenSecret
309 }
310
311 return ret
312 }
Signer会监听集群中secret的更新,然后通过secret的type判断其是否是boot token,然后更新到cluster-info中。目前我们遇到的现象是cluster-info中没有更新,说明监听器没有监听到新token secret的产生,但实际token创建已经成功。为此我们看一下controller-manager的日志看看其运行情况。