k8s部署 harbor

1.创建 Harbor 的命名空间：

kubectl create ns harbor

2.创建harbor秘钥

主harbor秘钥

mkdir -p /home/master/harbor_crt
cd  /home/master/harbor_crt

## 获得证书
openssl req -newkey rsa:4096 -nodes -sha256 -keyout ca.key -x509 -days 3650 -out ca.crt  -subj  "/C=CN/ST=Guangdong/L=Guangzhou/O=example/OU=example/CN=10.1.1.221"

## 生成证书签名请求
openssl req -newkey rsa:4096 -nodes -sha256 -keyout tls.key -out tls.csr  -subj  "/C=CN/ST=Guangdong/L=Guangzhou/O=example/OU=example/CN=10.1.1.221"


cat > extfile.cnf <<EOF
subjectAltName = IP:10.1.1.221
EOF


## 生成证书
openssl x509 -req -days 3650 -in tls.csr -CA ca.crt -CAkey ca.key -CAcreateserial  -extfile extfile.cnf -out tls.crt


#创建秘钥
cd  /home/master/harbor_crt

kubectl create secret generic harbor-tls --from-file=tls.crt --from-file=tls.key --from-file=ca.crt -n harbor

kubectl -n harbor get secret  harbor-tls

让系统信任我们的根证书（可选）
update-ca-trust extract 命令将PEM格式的根证书内容附加到/etc/ssl/certs/ca-certificates.crt ，而/etc/ssl/certs/ca-certificates.crt 包含了系统自带的各种可信根证书.

cp /home/master/harbor_crt/tls.crt /usr/local/share/ca-certificates

update-ca-trust extract 

3.添加 Harbor Helm Chart 仓库

#添加仓库
helm repo add harbor https://helm.goharbor.io 

#更新yum
helm repo update

#查看包
helm search repo harbor

从 Helm 仓库中拉取 Harbor Chart：

cd /home/master
#拉取包
helm pull harbor/harbor --version v1.11.1


#解压包
tar -zxvf harbor-1.11.1.tgz

cd harbor

4.配置 Harbor 的 values.yaml 文件

主Harbor配置
vim values.yaml

#expose.type="nodePort"			// 用NodePort访问、如果想用域名就改成Ingress
#expose.tls.enabled="false"		// 关闭证书
##第一处修改第一行，开始找
expose:
  type: nodePort
  tls:
    enabled: true
    certSource: secret
    auto:
      commonName: ""
    secret:
      secretName: "harbor-tls"
      notarySecretName: "harbor-tls"

#第二处修改
  nodePort:
    name: harbor
    ports:
      http:
        port: 80
        #nodePort: 30002
      https:
        port: 443
        nodePort: 30003
      notary:
        port: 4443
        #nodePort: 30004
  loadBalancer:
    name: harbor
    IP: ""
    ports:
      httpPort: 80
      httpsPort: 443
      notaryPort: 4443
    annotations: {}
    sourceRanges: []
#填写node节点ip
externalURL: https://10.1.1.221:30003


#第三处修改
#storageClass 填写动态存储 kubectl get sc
persistence:
  enabled: true
  resourcePolicy: "keep"
  persistentVolumeClaim:
    registry:
      existingClaim: ""
      storageClass: "nfs-boge"
      subPath: ""
      accessMode: ReadWriteMany
      size: 20Gi
      annotations: {}
    chartmuseum:
      existingClaim: ""
      storageClass: "nfs-boge"
      subPath: ""
      accessMode: ReadWriteMany
      size: 5Gi
      annotations: {}
    jobservice:
      jobLog:
        existingClaim: ""
        storageClass: "nfs-boge"
        subPath: ""
        accessMode: ReadWriteMany
        size: 5Gi
        annotations: {}
    database:
      existingClaim: ""
      storageClass: "nfs-boge"
      subPath: ""
      accessMode: ReadWriteMany
      size: 5Gi
      annotations: {}
    redis:
      existingClaim: ""
      storageClass: "nfs-boge"
      subPath: ""
      accessMode: ReadWriteMany
      size: 5Gi
      annotations: {}
    trivy:
      existingClaim: ""
      storageClass: "nfs-boge"
      subPath: ""
      accessMode: ReadWriteMany
      size: 5Gi
      annotations: {}


#replicas: 1
replicas: 3

5.安装harbor

#安装主harbor
cd /home/master/harbor
helm install harbor . -f values.yaml -n harbor




kubectl -n harbor get po

#访问url
https://10.1.1.221:30003
#默认用户：admin
#默认密码：Harbor12345

#查看
helm -n harbor ls
#卸载
helm uninstall harbor  -n harbor

6.服务器配置镜像仓库

因此要让docker信任我们的证书，为docker配置harbor证书
在/etc/docker目录下创建certs.d 文件夹，然后在 certs.d 文件夹下创建

mkdir -p /etc/docker/certs.d/

cd /home/master/harbor_crt


#转换tls.crt为tls.cert，供docker使用，Docker 守护进程将.crt文件解释为 CA 证书，将.cert文件解释为客户端证书
openssl x509 -inform PEM -in tls.crt -out tls.cert

#将前面创建了HTTPS的证书ca.crt、tls.cert、tls.key证书复制到（每一台docker主机都需要）
cp ca.crt /etc/docker/certs.d/
cp tls.key /etc/docker/certs.d/
cp tls.cert /etc/docker/certs.d/


scp -r  /etc/docker/certs.d/* root@k8s-node01:/etc/docker/certs.d/
scp -r  /etc/docker/certs.d/* root@k8s-node02:/etc/docker/certs.d/


cat > /etc/docker/daemon.json <<EOF
{
    "registry-mirrors": ["https://nr240upq.mirror.aliyuncs.com", "https://registry.docker-cn.com", "https://docker.mirrors.ustc.edu.cn", "https://dockerhub.azk8s.cn", "http://hub-mirror.c.163.com"],
  "max-concurrent-downloads": 10,
  "log-driver": "json-file",
  "log-level": "warn",
  "log-opts": {
    "max-size": "10m",
    "max-file": "3"
    },
  "insecure-registries":
        ["127.0.0.1"],
    "insecure-registries": ["https://10.1.1.221:30003","https://10.1.1.222:30003"],
  "data-root":"/var/lib/docker"
}
EOF
 
 
#重启docker
systemctl daemon-reload && systemctl restart docker

7.使用仓库

#登入仓库网站
docker login -u admin -p Harbor12345 10.1.1.221:30003

#下载镜像
docker pull wangyanglinux/myapp:v2
docker pull wangyanglinux/myapp:v3

#给镜像打上标签
docker tag wangyanglinux/myapp:v2 10.1.1.221:30003/library/myapp:v2
docker tag wangyanglinux/myapp:v3 10.1.1.221:30003/library/myapp:v3

#镜像上传
docker push 10.1.1.221:30003/library/myapp:v2
docker push 10.1.1.221:30003/library/myapp:v3

#删除镜像
docker rmi 10.1.1.221:30003/library/myapp:v2
docker rmi 10.1.1.221:30003/library/myapp:v3

#打包
docker save wangyanglinux/myapp:v2  > /root/myapp-v2.tar

#加载包
docker load -i /root/myapp-v2.tar


#使用kubernetes任一节点主机IP和30002端口即可访问UI管理界面。


#卸载harbor
kubectl delete ns harbor node-harbor --force
kubectl -n node-harbor get pv |grep node | awk 'NR!=1{print $1}' | xargs kubectl delete pv

rm -rf /nfs_dir/node-harbor-*

8.仓库管理-新建目标

复制管理-新建规则

测试复制