kubesphere+kubernetes搭建生产环境高可用集群（一）

等句晚安

已于 2022-06-30 09:09:09 修改

阅读量945

点赞数

文章标签： docker 网络 linux

于 2022-06-29 17:22:25 首次发布

本文链接：https://blog.csdn.net/weixin_45692576/article/details/125526097

版权

docker部署安装harbor镜像仓库（附证书制作）

节点角色	主机名	CPU(核)	内存(GB)	系统盘(GB)	数据盘(TB)	IP	备注
镜像仓库节点	harbor	2*8	256	2*600	10*3	10.4.11.40	harbor

harbor的安装部署

证书制作

#创建制作证书临时目录
[root@harbor ~]# /data/cert
[root@harbor ~]# cd /data/cert
#生成 CA 证书私钥。
[root@harbor cert]# openssl genrsa -out ca.key 4096

#生成 CA 证书。此处 -days可指定证书时间  mydockerhub.com为要访问的harbor域名
[root@harbor cert]# openssl req -x509 -new -nodes -sha512 -days 3650  -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=mydockerhub.com"  -key ca.key  -out ca.crt

#生成私钥。
[root@harbor cert]# openssl genrsa -out mydockerhub.com.key 4096

#生成证书签名请求 (CSR)。
#调整-subj选项中的值以反映您的组织。如果您使用 FQDN 连接您的 Harbor 主机，则必须将其指定为公用名称 ( CN) 属性并在密钥和 CSR 文件名中使用它。
[root@harbor cert]# openssl req -sha512 -new -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=mydockerhub.com"  -key mydockerhub.com.key -out mydockerhub.com.csr

#生成 x509 v3 扩展文件。
#无论您是使用 FQDN 还是 IP 地址连接到您的 Harbor 主机，您都必须创建此文件，以便为您的 Harbor 主机生成符合主题备用名称 (SAN) 和 x509 v3 的证书扩展要求。替换DNS条目以反映您的域。
[root@harbor cert]# cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names

[alt_names]
DNS.1=mydockerhub.com
DNS.2=mydockerhub
DNS.3=10.4.11.40
EOF

#使用该v3.ext文件为您的 Harbor 主机生成证书。
#将yourdomain.comCRS 和 CRT 文件名中的 替换为 Harbor 主机名。
[root@harbor cert]# openssl x509 -req -sha512 -days 3650 \
    -extfile v3.ext \
    -CA ca.crt -CAkey ca.key -CAcreateserial \
    -in mydockerhub.com.csr \
    -out mydockerhub.com.crt

#转换yourdomain.com.crt为yourdomain.com.cert, 供 Docker 使用。
#Docker 守护进程将.crt文件解释为 CA 证书，将.cert文件解释为客户端证书。
[root@harbor cert]# openssl x509 -inform PEM -in mydockerhub.com.crt -out mydockerhub.com.cert
#生生的证书如下
[root@harbor cert]# ls
ca.crt  ca.key  ca.srl  mydockerhub.com.cert  mydockerhub.com.crt  mydockerhub.com.csr  mydockerhub.com.key  v3.ext

安装docker-ce

# step 1: 安装必要的一些系统工具
[root@harbor ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
# Step 2: 添加软件源信息
[root@harbor ~]# yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# Step 3
[root@harbor ~]# sed -i 's+download.docker.com+mirrors.aliyun.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo
# Step 4: 更新并安装Docker-CE
[root@harbor ~]# yum makecache fast
[root@harbor ~]# yum -y install docker-ce
# 安装指定版本的Docker-CE:
# Step 1: 查找Docker-CE的版本:
# yum list docker-ce.x86_64 --showduplicates | sort -r
#   Loading mirror speeds from cached hostfile
#   Loaded plugins: branch, fastestmirror, langpacks
#   docker-ce.x86_64            17.03.1.ce-1.el7.centos            docker-ce-stable
#   docker-ce.x86_64            17.03.1.ce-1.el7.centos            @docker-ce-stable
#   docker-ce.x86_64            17.03.0.ce-1.el7.centos            docker-ce-stable
#   Available Packages
# Step2: 安装指定版本的Docker-CE: (VERSION例如上面的17.03.0.ce.1-1.el7.centos)
# yum -y install docker-ce-[VERSION]

配置并启动docker

#创建docker配置文件目录
[root@harbor ~]# mkdir /etc/docker/
#编辑配置文件，修改docker存储目录，并配置日志限制（可选）
[root@harbor ~]# vi /etc/docker/daemon.json
{
    "data-root": "/data/disk01/docker",
    "log-driver":"json-file",
    "log-opts": {"max-size":"500m", "max-file":"7"}
}
[root@harbor ~]# cd /etc/docker/
[root@harbor docker]# mkdir -p ./certs.d/mydockerhub.com:18443
[root@harbor docker]# cp /data/cert/mydockerhub.com.cert /data/cert/mydockerhub.com.key /data/cert/ca.crt ./certs.d/mydockerhub.com:18443/

#目录结构如下
/etc/docker/certs.d/
    └── mydockerhub.com:18443
       ├── mydockerhub.com.cert  <-- Server certificate signed by CA
       ├── mydockerhub.com.key   <-- Server key signed by CA
       └── ca.crt
       
#启动docker，并配置开机自启
[root@harbor dockerhub.dsj.com:18443]# systemctl start docker && systemctl enable docker
Created symlink from /etc/systemd/system/multi-user.target.wants/docker.service to /usr/lib/systemd/system/docker.service.
#查看docker服务状态
[root@harbor dockerhub.dsj.com:18443]# systemctl status docker

下载安装docker-compose

#到下面的地址找到相应版本下载
https://github.com/docker/compose/releases
#下载并将docker-compose添加到全局命令
curl -L "https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
#查看docker-compose版本
docker-compose version

配置安装harbor(v2.4.1)

#上次harbor离线包 harbor-offline-installer-v2.4.1.tgz 
[root@harbor ~]# tar xf  harbor-offline-installer-v2.4.1.tgz  -C /data
[root@harbor ~]# cd /data/harbor
#创建证书存储目录
[root@harbor harbor]# mkdir cert
#拷贝证书文件
[root@harbor harbor]# cp /data/cert/mydockerhub.com.crt /data/cert/mydockerhub.com.key ./cert/
[root@harbor harbor]# cp harbor.yml.tmpl harbor.yml
#修改harbor安装配置文件，主要修改以下内容
[root@harbor harbor]# vi harbor.yml

# Configuration file of Harbor

# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: mydockerhub.com #harbor的域名

# http related config
http:
  # port for http, default is 80. If https enabled, this port will redirect to https port
  port: 18080 #http协议端口，设置https协议后会自动跳转到external_url

# https related config
https:
  # https port for harbor, default is 443
  port: 18443 #https协议的端口
  # The path of cert and key files for nginx
  certificate: /data/disk01/harbor/cert/mydockerhub.com.crt #crt证书路径
  private_key: /data/disk01/harbor/cert/mydockerhub.com.key #key证书路径

# # Uncomment following will enable tls communication between all harbor components
# internal_tls:
#   # set enabled to true means internal tls is enabled
#   enabled: true
#   # put your cert and key files on dir
#   dir: /etc/harbor/tls/internal

# Uncomment external_url if you want to enable external proxy
# And when it enabled the hostname will no longer used
external_url: https://mydockerhub.com:18443 #harbor的访问url

# The initial password of Harbor admin
# It only works in first time to install harbor
# Remember Change the admin password from UI after launching Harbor.
harbor_admin_password: Harbor_123 #harbor的admin密码

# Harbor DB configuration
database:
  # The password for the root user of Harbor DB. Change this before any production use.
  password: root_123 #harbor的数据库root密码
  # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
  max_idle_conns: 100
  # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
  # Note: the default number of connections is 1024 for postgres of harbor.
  max_open_conns: 900

# The default data volume
data_volume: /data/disk01/harbor_data #harbor的数据存储目录

# Harbor Storage settings by default is using /data dir on local filesystem
# Uncomment storage_service setting If you want to using external storage
# storage_service:
#   # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
#   # of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
#   ca_bundle:

#   # storage backend, default is filesystem, options include filesystem, azure, gcs, s3, swift and oss
#   # for more info about this configuration please refer https://docs.docker.com/registry/configuration/
#   filesystem:
#     maxthreads: 100
#   # set disable to true when you want to disable registry redirect

.................................


#导入harbor所需的docker镜像
[root@harbor harbor]# docker load -i harbor.v2.4.1.tar.gz
#运行安装脚本
[root@harbor harbor]# ./prepare
[root@harbor harbor]# ./install.sh

[Step 0]: checking if docker is installed ...

Note: docker version: 20.10.12

[Step 1]: checking docker-compose is installed ...

Note: docker-compose version: 1.29.1

[Step 2]: loading Harbor images ...
Loaded image: goharbor/registry-photon:v2.4.1
Loaded image: goharbor/notary-signer-photon:v2.4.1
Loaded image: goharbor/harbor-core:v2.4.1
Loaded image: goharbor/redis-photon:v2.4.1
Loaded image: goharbor/harbor-jobservice:v2.4.1
Loaded image: goharbor/harbor-registryctl:v2.4.1
Loaded image: goharbor/nginx-photon:v2.4.1
Loaded image: goharbor/notary-server-photon:v2.4.1
Loaded image: goharbor/harbor-log:v2.4.1
Loaded image: goharbor/harbor-db:v2.4.1
Loaded image: goharbor/harbor-exporter:v2.4.1
Loaded image: goharbor/trivy-adapter-photon:v2.4.1
Loaded image: goharbor/chartmuseum-photon:v2.4.1
Loaded image: goharbor/prepare:v2.4.1
Loaded image: goharbor/harbor-portal:v2.4.1


[Step 3]: preparing environment ...

[Step 4]: preparing harbor configs ...
prepare base dir is set to /data/disk01/harbor
Clearing the configuration file: /config/log/rsyslog_docker.conf
Clearing the configuration file: /config/log/logrotate.conf
Clearing the configuration file: /config/registry/config.yml
Clearing the configuration file: /config/registry/passwd
Clearing the configuration file: /config/db/env
Clearing the configuration file: /config/jobservice/config.yml
Clearing the configuration file: /config/jobservice/env
Clearing the configuration file: /config/portal/nginx.conf
Clearing the configuration file: /config/nginx/nginx.conf
Clearing the configuration file: /config/registryctl/config.yml
Clearing the configuration file: /config/registryctl/env
Clearing the configuration file: /config/core/app.conf
Clearing the configuration file: /config/core/env
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
loaded secret from file: /data/secret/keys/secretkey
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir



[Step 5]: starting Harbor ...
Creating network "harbor_harbor" with the default driver
Creating harbor-log ... done
Creating redis         ... done
Creating harbor-portal ... done
Creating harbor-db     ... done
Creating registry      ... done
Creating registryctl   ... done
Creating harbor-core   ... done
Creating harbor-jobservice ... done
Creating nginx             ... done
✔ ----Harbor has been installed and started successfully.----

#查看harbor的状态
[root@harbor harbor]# docker-compose ps

      Name                     Command                  State                                               Ports
----------------------------------------------------------------------------------------------------------------------------------------------------------
harbor-core         /harbor/entrypoint.sh            Up (healthy)
harbor-db           /docker-entrypoint.sh 96 13      Up (healthy)
harbor-jobservice   /harbor/entrypoint.sh            Up (healthy)
harbor-log          /bin/sh -c /usr/local/bin/ ...   Up (healthy)   127.0.0.1:1514->10514/tcp
harbor-portal       nginx -g daemon off;             Up (healthy)
nginx               nginx -g daemon off;             Up (healthy)   0.0.0.0:18080->8080/tcp,:::18080->8080/tcp, 0.0.0.0:18443->8443/tcp,:::18443->8443/tcp
redis               redis-server /etc/redis.conf     Up (healthy)
registry            /home/harbor/entrypoint.sh       Up (healthy)
registryctl         /home/harbor/start.sh            Up (healthy)

#访问测试
https://10.4.11.40:18443/
用户名：admin
密码：Harbor_123
#根据需要创建所需项目及用户
例：创建名为kubesphere、grafana、thanosio、calico的项目用来存储kubesphere部署所需镜像，并创建kubesphere用户授权为该项目的项目管理员角色