k3s-ansible 集群离线，私有镜像库

第一准备阶段

 

1.下载k3s-ansible

修改inventory/jettech-cluster/hosts.ini 里面的ip

基于ssh-keygen ssh-copy-id

也可以基于密码方式 我用的是基于ssh

[root@jettoloader work]# ssh-keygen

[root@jettoloader work]# ssh-copy-id -i ~/.ssh/id_rsa.pub 172.16.10.15

[root@localhost test]# ansible-playbook reset_role.yml -i inventory/jettech-cluster/hosts.ini -u root -b 清理

[root@localhost test]# ansible-playbook site_role.yml -i inventory/jettech-cluster/hosts.ini -u root -b 部署

git clone https://github.com/wubomichael/k3s-ansible.git

2.进入目录

cd k3s-ansible-master/roles/copy/files

3. 去github下载相应的版本文件k3s和k3s-airgap-images-amd64.tar 这个是镜像列表 k3s-images.txt

https://github.com/k3s-io/k3s/releases

4.所有节点需要，因为k3s 去下载镜像 

[root@localhost files]# cat /etc/rancher/k3s/registries.yaml 
mirrors:
  "172.16.10.5:5000":
    endpoint:
      - "http://172.16.10.5:5000"

5.弄一个私有镜像库harbor或docker.io/library/registry:latest 都可以

[root@localhost files]# mkdir /opt/registry
[root@localhost files]# k3s ctr  run --null-io --net-host -mount type=bind,src=/opt/registry,dst=/var/lib/registry,options=rbind:rw   -d docker.io/library/registry:latest jettech-registry

6.docker镜像转换到ctr 

pull
[root@localhost test]# docker pull  harbor.jettech.com/library/busybox:1.28.4
1.28.4: Pulling from library/busybox
07a152489297: Pull complete 
Digest: sha256:74f634b1bc1bd74535d5209589734efbd44a25f4e2dc96d78784576a3eb5b335
Status: Downloaded newer image for harbor.jettech.com/library/busybox:1.28.4
harbor.jettech.com/library/busybox:1.28.4

save
[root@localhost test]# docker save harbor.jettech.com/library/busybox:1.28.4 > busybox.tar

导入
[root@localhost test]# k3s ctr i import busybox.tar
unpacking harbor.jettech.com/library/busybox:1.28.4 (sha256:585093da3a716161ec2b2595011051a90d2f089bc2a25b4a34a18e2cf542527c)...done

tag：
[root@localhost test]# k3s ctr i tag harbor.jettech.com/library/busybox:1.28.4 172.16.10.5:5000/library/busybox:1.28.4
172.16.10.5:5000/library/busybox:1.28.4

push
[root@localhost test]# k3s ctr i push --plain-http 172.16.10.5:5000/library/busybox:1.28.4
manifest-1.28.4@sha256:585093da3a716161ec2b2595011051a90d2f089bc2a25b4a34a18e2cf542527c: done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:432b65032b9466b4dadcc5c7b11701e71d21c18400aae946b101ad16be62333a:           done           |++++++++++++++++++++++++++++++++++++++| 
config-sha256:8c811b4aec35f259572d0f79207bc0678df4c736eeec50bc9fec37ed936a472a:          done           |++++++++++++++++++++++++++++++++++++++| 
elapsed: 0.2 s                                                                           total:  1.3 Mi (6.5 MiB/s)

或直接从harbor下载也可以

我的harbor是https方式的
[root@localhost test]# k3s ctr i pull --user admin:Harbor12345  harbor.jettech.com/library/busybox:1.28.4
harbor.jettech.com/library/busybox:1.28.4:                                        resolved       |++++++++++++++++++++++++++++++++++++++| 
manifest-sha256:74f634b1bc1bd74535d5209589734efbd44a25f4e2dc96d78784576a3eb5b335: exists         |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:07a152489297fc2bca20be96fab3527ceac5668328a30fd543a160cd689ee548:    exists         |++++++++++++++++++++++++++++++++++++++| 
config-sha256:8c811b4aec35f259572d0f79207bc0678df4c736eeec50bc9fec37ed936a472a:   exists         |++++++++++++++++++++++++++++++++++++++| 
elapsed: 0.3 s                                                                    total:   0.0 B (0.0 B/s)                                         
unpacking linux/amd64 sha256:74f634b1bc1bd74535d5209589734efbd44a25f4e2dc96d78784576a3eb5b335...
done: 16.581871ms

7.列子nginx

[root@localhost test]# cat nginx.yaml 
apiVersion: v1
kind: Service
metadata:
  labels: {name: nginx}
  name: nginx
spec:
  ports:
  - {name: t9080, nodePort: 30002, port: 80, protocol: TCP, targetPort: 80}
  selector: {name: nginx}
  type: NodePort
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels: {name: nginx}
spec:
  replicas: 1
  selector:
    matchLabels: {name: nginx}
  template:
    metadata:
      name: nginx
      labels: {name: nginx}
    spec:
      containers:
      - name: nginx
        #image: harbor.jettech.com/jettechtools/nginx:1.21.4
        image: 172.16.10.5:5000/library/nginx:1.21.4
        #volumeMounts:
        #- name: volv
        #  mountPath: /data 
        #image: nginx:1.21.4
      #volumes:
      #- name: volv
      #  persistentVolumeClaim:
      #    claimName: local-path-pvc

busybox: 

[root@localhost test]# cat busybox.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: busybox
  labels: {name: busybox}
spec:
  replicas: 1
  selector:
    matchLabels: {name: busybox}
  template:
    metadata:
      name: busybox
      labels: {name: busybox}
    spec:
      containers:
      - name: busybox
        #image: harbor.jettech.com/library/busybox:1.28.4
        image: 172.16.10.5:5000/library/busybox:1.28.4
        #image: busybox:1.28.4
        args:
        - /bin/sh
        - -c
        - sleep 10; touch /tmp/healthy; sleep 30000
        readinessProbe:           #就绪探针
          exec:
            command:
            - cat
            - /tmp/healthy
          initialDelaySeconds: 10         #10s之后开始第一次探测
          periodSeconds: 5

查看节点，发现 ROLES为none 

[root@localhost ~]# kubectl get  nodes
NAME           STATUS   ROLES                  AGE   VERSION
172.16.10.5    Ready    control-plane,master   22m   v1.22.5+k3s1
172.16.10.15   Ready    <none>                 21m   v1.22.5+k3s1
172.16.10.21   Ready    <none>                 22m   v1.22.5+k3s1

 修改

修改
[root@jettoloader k3s-ansible-master]# k3s kubectl label  node/172.16.10.15 node-role.kubernetes.io/worker=worker
node/172.16.10.15 labeled
[root@jettoloader k3s-ansible-master]# k3s kubectl label  node/172.16.10.21 node-role.kubernetes.io/worker=worker
node/172.16.10.21 labeled

查看
[root@localhost ~]# kubectl get  nodes
NAME           STATUS   ROLES                  AGE   VERSION
172.16.10.5    Ready    control-plane,master   24m   v1.22.5+k3s1
172.16.10.15   Ready    worker                 24m   v1.22.5+k3s1
172.16.10.21   Ready    worker                 24m   v1.22.5+k3s1


删除
[root@jettoloader k3s-ansible-master]# k3s kubectl label  node/172.16.10.15 node-role.kubernetes.io/worker-

删除后查看
[root@localhost ~]# kubectl get  nodes
NAME           STATUS   ROLES                  AGE   VERSION
172.16.10.5    Ready    control-plane,master   24m   v1.22.5+k3s1
172.16.10.15   Ready    worker                 24m   v1.22.5+k3s1
172.16.10.21   Ready    <none>                 24m   v1.22.5+k3s1

8.启动

[root@localhost test]# kubectl create -f busybox.yaml -f nginx.yaml

[root@localhost test]# kubectl get pods,svc  -o wide -A
NAMESPACE     NAME                                         READY   STATUS      RESTARTS   AGE   IP          NODE           NOMINATED NODE   READINESS GATES
kube-system   pod/metrics-server-9cf544f65-95d4n           1/1     Running     0          37m   10.42.0.5   172.16.10.5    <none>           <none>
kube-system   pod/local-path-provisioner-64ffb68fd-j7khw   1/1     Running     0          37m   10.42.0.6   172.16.10.5    <none>           <none>
kube-system   pod/helm-install-traefik-crd--1-bntlv        0/1     Completed   0          37m   10.42.0.2   172.16.10.5    <none>           <none>
kube-system   pod/helm-install-traefik--1-v8vzg            0/1     Completed   1          37m   10.42.0.4   172.16.10.5    <none>           <none>
kube-system   pod/svclb-traefik-r4dqj                      2/2     Running     0          36m   10.42.0.7   172.16.10.5    <none>           <none>
kube-system   pod/svclb-traefik-dp2zc                      2/2     Running     0          36m   10.42.1.5   172.16.10.15   <none>           <none>
kube-system   pod/coredns-85cb69466-rlkd4                  1/1     Running     0          37m   10.42.0.3   172.16.10.5    <none>           <none>
kube-system   pod/traefik-786ff64748-dw4vv                 1/1     Running     0          36m   10.42.0.8   172.16.10.5    <none>           <none>
default       pod/nginx-7ccc7d869c-99cz7                   1/1     Running     0          18m   10.42.1.8   172.16.10.15   <none>           <none>
default       pod/busybox-f8fb8584-rk9ft                   0/1     Running     0          6s    10.42.1.9   172.16.10.15   <none>           <none>

NAMESPACE     NAME                     TYPE           CLUSTER-IP     EXTERNAL-IP                PORT(S)                     AGE   SELECTOR
default       service/kubernetes       ClusterIP      10.43.0.1      <none>                     443/TCP                     37m   <none>
kube-system   service/kube-dns         ClusterIP      10.43.0.254    <none>                     53/UDP,53/TCP,9153/TCP      37m   k8s-app=kube-dns
kube-system   service/metrics-server   ClusterIP      10.43.40.81    <none>                     443/TCP                     37m   k8s-app=metrics-server
kube-system   service/traefik          LoadBalancer   10.43.77.227   172.16.10.15,172.16.10.5   80:1712/TCP,443:44779/TCP   36m   app.kubernetes.io/instance=traefik,app.kubernetes.io/name=traefik
default       service/nginx            NodePort       10.43.89.112   <none>                     80:30002/TCP                18m   name=nginx

9.测试nginx服务和dns

[root@localhost test]# kubectl exec -it pod/busybox-f8fb8584-rk9ft -n default sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
/ # nslookup nginx
Server:    10.43.0.254
Address 1: 10.43.0.254 kube-dns.kube-system.svc.jettech.com

Name:      nginx
Address 1: 10.43.89.112 nginx.default.svc.jettech.com
/ # nslookup nginx.default.svc.jettech.com
Server:    10.43.0.254
Address 1: 10.43.0.254 kube-dns.kube-system.svc.jettech.com

Name:      nginx.default.svc.jettech.com
Address 1: 10.43.89.112 nginx.default.svc.jettech.com