1.Docker仓库harbor高可用
实验环境
| 节点 | IP | 域名 |
|---|---|---|
| harbor25 | 172.16.62.25 | harbor25.haostack.com |
| harbor26 | 172.16.62.26 | harbor.haostack.com |
| DNS | 172.16.62.24 |
1.harbor26主机安装harbor
1.1.解压安装包
tar -xvf harbor-offline-installer-v1.7.5.tgz -C /usr/local/src
1.2.harbor.cfg配置
- 根据需求修改，域名为 harbor.haostack.com 已经做解析
[root@harbor26 harbor]# more harbor.cfg
## Configuration file of Harbor
#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
_version = 1.7.0
#The IP address or hostname to access admin UI and registry service.
#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail.
hostname = harbor.haostack.com #修改域名访问
#The protocol for accessing the UI and token/notification service, by default it is http.
#It can be set to https if ssl is enabled on nginx.
ui_url_protocol = http
#Maximum number of job workers in job service
max_job_workers = 10
#Determine whether or not to generate certificate for the registry's token.
#If the value is on, the prepare script creates new root cert and private key
#for generating token to access the registry. If the value is off the default key/cert will be used.
#This flag also controls the creation of the notary signer's cert.
customize_crt = on
#The path of cert and key files for nginx, they are applied only the protocol is set to https
ssl_cert = /data/cert/server.crt #指向crt目录
ssl_cert_key = /data/cert/server.key #指向key目录
#The path of secretkey storage
secretkey_path = /data #key目录
#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone
admiral_url = NA
#Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
1.3.执行脚本安装./install.sh
[root@harbor26 harbor]# ./install.sh
[Step 0]: checking installation environment ...
Note: docker version: 19.03.11
✖ Need to install docker-compose(1.7.1+) by yourself first and run this script again.
[root@harbor26 harbor]#
#提示需要安装docker compose
1.4.安装docker-compose
- 需要提前准备好epel.repo
yum install docker-compose
15.再次执行脚本安装./install.sh
Creating harbor-adminserver ... done
Creating harbor-core ... done
[Step 4]: starting Harbor ...
Creating harbor-portal ... done
Creating nginx ... done
Creating registryctl ...
Creating registry ...
Creating harbor-db ...
Creating harbor-adminserver ...
Creating redis ...
Creating harbor-core ...
Creating harbor-portal ...
Creating harbor-jobservice ...
Creating nginx ...
✔ ----Harbor has been installed and started successfully.----
Now you should be able to visit the admin portal at http://harbor.haostack.com.
For more details, please visit https://github.com/goharbor/harbor .
#查看docker-compose
[root@harbor26 ~]# docker-compose version
docker-compose version 1.18.0, build 8dd22a9
docker-py version: 2.6.1
CPython version: 3.6.8
OpenSSL version: OpenSSL 1.0.2k-fips 26 Jan 2017
[root@harbor26 ~]#
1.6 安装完成
[root@harbor26 ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
f3cc403571d2 goharbor/nginx-photon:v1.7.5 "nginx -g 'daemon of…" 6 weeks ago Up 6 weeks (healthy) 0.0.0.0:80->80/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:4443->4443/tcp nginx
93e933be9ef3 goharbor/harbor-portal:v1.7.5 "nginx -g 'daemon of…" 6 weeks ago Up 6 weeks (healthy) 80/tcp harbor-portal
f66be3c9b2ac goharbor/harbor-jobservice:v1.7.5 "/harbor/start.sh" 6 weeks ago Up 6 weeks harbor-jobservice
cc138a760e96 goharbor/harbor-core:v1.7.5 "/harbor/start.sh" 6 weeks ago Up 6 weeks (healthy) harbor-core
11cb7e87a576 goharbor/harbor-db:v1.7.5 "/entrypoint.sh post…" 6 weeks ago Up 6 weeks (healthy) 5432/tcp harbor-db
32a558c54ea5 goharbor/harbor-adminserver:v1.7.5 "/harbor/start.sh" 6 weeks ago Up 6 weeks (healthy) harbor-adminserver
68eceea988b5 goharbor/registry-photon:v2.6.2-v1.7.5 "/entrypoint.sh /etc…" 6 weeks ago Up 6 weeks (healthy) 5000/tcp registry
3b1d7ef4fb36 goharbor/redis-photon:v1.7.5 "docker-entrypoint.s…" 6 weeks ago Up 6 weeks 6379/tcp redis
9f79b4061144 goharbor/harbor-registryctl:v1.7.5 "/harbor/start.sh" 6 weeks ago Up 6 weeks (healthy) registryctl
ef6d75c54679 goharbor/harbor-log:v1.7.5 "/bin/sh -c /usr/loc…" 6 weeks ago Up 6 weeks (healthy) 127.0.0.1:1514->10514/tcp harbor-log
[root@harbor26 ~]#
1.7.登录harbor
- 用户名 admin
- 密码 Harbor12345 (默认)
1.8.客户端访问
#客户端需要更改内部仓库地址
//修改docker脚本启动参数
[root@node13 docker_data]# more /lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
BindsTo=containerd.service
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --insecure-registry=harbor.haostack.com
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
1.9.重启服务
[root@harbor26 harbor]# systemctl daemon-reload
[root@harbor26 harbor]# systemctl restart docker
[root@harbor26 harbor]#
1.10.上传镜像
//先登录仓库 harbor.haostack.com
root@harbor25 tmp]# docker login harbor.haostack.com
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@harbor25 tmp]#
#给镜像打tag
root@harbor25 tmp]# docker tag nginx:v1 harbor.haostack.com/baseimages/nginx:v1
[root@harbor25 tmp]# docker image ls
REPOSITORY TAG IMAGE ID CREATED SIZE
nginx latest 4392e5dad77d 39 hours ago 132MB
nginx v1 4392e5dad77d 39 hours ago 132MB
harbor.haostack.com/baseimages/nginx v1 4392e5dad77d 39 hours ago 132MB
//上传镜像到harbor上
[root@harbor25 tmp]# docker push harbor.haostack.com/baseimages/nginx:v1
The push refers to repository [harbor.haostack.com/baseimages/nginx]
787328500ad5: Pushed
077ae58ac205: Pushed
8c7fd6263c1f: Pushed
d9c0b16c8d5b: Pushed
ffc9b21953f4: Pushed
v1: digest: sha256:d9002da0297bcd0909b394c26bd0fc9d8c466caf2b7396f58948cac5318d0d0b size: 1362
1. 11.验证镜像
1.12 下载镜像
- 从其他客户端测试下载镜像
-#需要先修改配置文件指向 harbor.haostack.com
After=network-online.target firewalld.service containerd.service
Wants=network-online.target
Requires=docker.socket
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --insecure-registry=harbor.haostack.com
ExecReload=/bin/kill -s HUP $MAINPID
TimeoutSec=0
RestartSec=2
Restart=always
[root@harbor25 harbor]# docker pull harbor.haostack.com/baseimages/nginx:v1
v1: Pulling from baseimages/nginx
afb6ec6fdc1c: Pull complete
dd3ac8106a0b: Pull complete
8de28bdda69b: Pull complete
a2c431ac2669: Pull complete
e070d03fd1b5: Pull complete
Digest: sha256:d9002da0297bcd0909b394c26bd0fc9d8c466caf2b7396f58948cac5318d0d0b
Status: Downloaded newer image for harbor.haostack.com/baseimages/nginx:v1
harbor.haostack.com/baseimages/nginx:v1
//查看已经下载成功
[root@harbor25 harbor]# docker image ls
REPOSITORY TAG IMAGE ID CREATED SIZE
harbor.haostack.com/baseimages/nginx v1 4392e5dad77d 39 hours ago 132MB
goharbor/chartmuseum-photon v0.8.1-v1.7.5 ca910ae935e8 14 months ago 113MB
goharbor/harbor-migrator v1.7.5 326685e45678 14 months ago 679MB
goharbor/redis-photon v1.7.5 6886d1b5b009 14 months ago 101MB
goharbor/clair-photon v2.0.8-v1.7.5 970bdadc8995 14 months ago 164MB
goharbor/notary-server-photon v0.6.1-v1.7.5 667b2f0fb68d 14 months ago 135MB
goharbor/notary-signer-photon v0.6.1-v1.7.5 9ea20ae211bc 14 months ago 132MB
goharbor/harbor-registryctl v1.7.5 b5781fe85ba3 14 months ago 102MB
goharbor/registry-photon v2.6.2-v1.7.5 789d7a921303 14 months ago 86.7MB
goharbor/nginx-photon v1.7.5 172196970055 14 months ago 35.9MB
goharbor/harbor-log v1.7.5 26784a32e151 14 months ago 81.4MB
goharbor/harbor-jobservice v1.7.5 0755114a9c68 14 months ago 84.1MB
goharbor/harbor-core v1.7.5 86925c0ee5d5 14 months ago 95.6MB
goharbor/harbor-portal v1.7.5 9a9082e62030 14 months ago 40.6MB
goharbor/harbor-adminserver v1.7.5 b2afe190593f 14 months ago 72.3MB
goharbor/harbor-db v1.7.5 c1226370ffaa 14 months ago 138MB
[root@harbor25 harbor]#
2.harbor25主机安装harbor
2.1 开始安装
tar -xvf harbor-offline-installer-v1.7.5.tgz -C /usr/local/src
2.2 harbor.cnf配置
- 根据需求修改，域名为 harbor25.haostack.com 已经做解析
[root@harbor25 certs]# more /usr/local/src/harbor/harbor.cfg
## Configuration file of Harbor
#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
_version = 1.7.0
#The IP address or hostname to access admin UI and registry service.
#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail.
hostname = harbor25.haostack.com #指向域名
#The protocol for accessing the UI and token/notification service, by default it is http.
#It can be set to https if ssl is enabled on nginx.
ui_url_protocol = https #https访问
#Maximum number of job workers in job service
max_job_workers = 10
#Determine whether or not to generate certificate for the registry's token.
#If the value is on, the prepare script creates new root cert and private key
#for generating token to access the registry. If the value is off the default key/cert will be used.
#This flag also controls the creation of the notary signer's cert.
customize_crt = on
#The path of cert and key files for nginx, they are applied only the protocol is set to https
#ssl_cert = /data/cert/server.crt
ssl_cert = /usr/local/src/harbor/certs/harbor-ca.crt
#ssl_cert_key = /data/cert/server.key #证书目录
ssl_cert_key = /usr/local/src/harbor/certs/harbor-ca.key #证书目录
#The path of secretkey storage
secretkey_path = /usr/local/src/harbor/certs
#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone
admiral_url = NA
#Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
log_rotate_count = 50
#Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
#If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
#are all valid.
log_rotate_size = 200M
#Config http proxy for Clair, e.g. http://my.proxy.com:3128
#Clair doesn't need to connect to harbor internal components via http proxy.
http_proxy =
https_proxy =
no_proxy = 127.0.0.1,localhost,core,registry
#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
#only take effect in the first boot, the subsequent changes of these properties
#should be performed on web ui
#************************BEGIN INITIAL PROPERTIES************************
#Email account settings for sending out password resetting emails.
#Email server uses the given username and password to authenticate on TLS connections to host and act as identity.
#Identity left blank to act as username.
email_identity =
email_server = smtp.mydomain.com
email_server_port = 25
email_username = sample_admin@mydomain.com
email_password = abc
email_from = admin <sample_admin@mydomain.com>
email_ssl = false
email_insecure = false
##The initial password of Harbor admin, only works for the first time when Harbor starts.
#It has no effect after the first launch of Harbor.
#Change the admin password from UI after launching Harbor.
harbor_admin_password = Harbor12345
##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
#Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
auth_mode = db_auth
#The url for an ldap endpoint.
ldap_url = ldaps://ldap.mydomain.com
#A user's DN who has the permission to search the LDAP/AD server.
#If your LDAP/AD server does not support anonymous search, you should configure this DN and ldap_search_pwd.
#ldap_searchdn = uid=searchuser,ou=people,dc=mydomain,dc=com
#the password of the ldap_searchdn
#ldap_search_pwd = password
#The base DN from which to look up a user in LDAP/AD
ldap_basedn = ou=people,dc=mydomain,dc=com
#Search filter for LDAP/AD, make sure the syntax of the filter is correct.
#ldap_filter = (objectClass=person)
# The attribute used in a search to match a user, it could be uid, cn, email, sAMAccountName or other attributes depending on your LDAP/AD
ldap_uid = uid
#the scope to search for users, 0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE
ldap_scope = 2
#Timeout (in seconds) when connecting to an LDAP Server. The default value (and most reasonable) is 5 seconds.
ldap_timeout = 5
#Verify certificate from LDAP server
ldap_verify_cert = true
#The base dn from which to lookup a group in LDAP/AD
ldap_group_basedn = ou=group,dc=mydomain,dc=com
#filter to search LDAP/AD group
ldap_group_filter = objectclass=group
#The attribute used to name a LDAP/AD group, it could be cn, name
ldap_group_gid = cn
#The scope to search for ldap groups. 0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE
ldap_group_scope = 2
#Turn on or off the self-registration feature
self_registration = on
#The expiration time (in minute) of token created by token service, default is 30 minutes
token_expiration = 30
#The flag to control what users have permission to create projects
#The default value "everyone" allows everyone to creates a project.
#Set to "adminonly" so that only admin user can create project.
project_creation_restriction = everyone
#************************END INITIAL PROPERTIES************************
#######Harbor DB configuration section#######
#The address of the Harbor database. Only need to change when using external db.
db_host = postgresql
#The password for the root user of Harbor DB. Change this before any production use.
db_password = root123
#The port of Harbor database host
db_port = 5432
#The user name of Harbor database
db_user = postgres
##### End of Harbor DB configuration#######
##########Redis server configuration.############
#Redis connection address
redis_host = redis
#Redis connection port
redis_port = 6379
#Redis connection password
redis_password =
#Redis connection db index
#db_index 1,2,3 is for registry, jobservice and chartmuseum.
#db_index 0 is for UI, it's unchangeable
redis_db_index = 1,2,3
########## End of Redis server configuration ############
##########Clair DB configuration############
#Clair DB host address. Only change it when using an exteral DB.
clair_db_host = postgresql
#The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair.
#Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database.
clair_db_password = root123
#Clair DB connect port
clair_db_port = 5432
#Clair DB username
clair_db_username = postgres
#Clair default database
clair_db = postgres
#The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
clair_updaters_interval = 12
##########End of Clair DB configuration############
#The following attributes only need to be set when auth mode is uaa_auth
uaa_endpoint = uaa.mydomain.org
uaa_clientid = id
uaa_clientsecret = secret
uaa_verify_cert = true
uaa_ca_cert = /path/to/ca.pem
### Harbor Storage settings ###
#Please be aware that the following storage settings will be applied to both docker registry and helm chart repository.
#registry_storage_provider can be: filesystem, s3, gcs, azure, etc.
registry_storage_provider_name = filesystem
#registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2".
#To avoid duplicated configurations, both docker registry and chart repository follow the same storage configuration specifications of docker registry.
#Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration.
registry_storage_provider_config =
#registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
#of registry's and chart repository's containers. This is usually needed when the user hosts a internal storage with self signed certificate.
registry_custom_ca_bundle =
#If reload_config=true, all settings which present in harbor.cfg take effect after prepare and restart harbor, it overwrites exsiting settings.
#reload_config=true
#Regular expression to match skipped environment variables
#skip_reload_env_pattern=(^EMAIL.*)|(^LDAP.*)
[root@harbor25 certs]#
#安装
Generated configuration file: ./common/config/core/app.conf
Creating harbor-log ... done
The configuration files are ready, please use docker-compose to start the service.
[Step 3]: checking existing instance of Harbor ...
Creating registry ... done
Creating harbor-core ... done
[Step 4]: starting Harbor ...
Creating harbor-portal ... done
Creating nginx ... done
Creating registryctl ...
Creating harbor-db ...
Creating harbor-adminserver ...
Creating registry ...
Creating redis ...
Creating harbor-core ...
Creating harbor-jobservice ...
Creating harbor-portal ...
Creating nginx ...
✔ ----Harbor has been installed and started successfully.----
Now you should be able to visit the admin portal at http://harbor25.haostack.com.
For more details, please visit https://github.com/goharbor/harbor .
#查看docker-compose
2.3 安装完成
- 查看容器
[root@harbor25 certs]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
e4199abf0fa3 goharbor/nginx-photon:v1.7.5 "nginx -g 'daemon of…" 6 weeks ago Up 6 weeks (healthy) 0.0.0.0:80->80/tcp, 0.0.0.0:443->443/tcp, 0.0.0.0:4443->4443/tcp nginx
8bdb5a7bef5f goharbor/harbor-portal:v1.7.5 "nginx -g 'daemon of…" 6 weeks ago Up 6 weeks (healthy) 80/tcp harbor-portal
7c865b845c60 goharbor/harbor-jobservice:v1.7.5 "/harbor/start.sh" 6 weeks ago Up 6 weeks harbor-jobservice
5e53bf0da3f8 goharbor/harbor-core:v1.7.5 "/harbor/start.sh" 6 weeks ago Up 6 weeks (healthy) harbor-core
472a7329034a goharbor/harbor-db:v1.7.5 "/entrypoint.sh post…" 6 weeks ago Up 6 weeks (healthy) 5432/tcp harbor-db
e2a8f5e6faa0 goharbor/harbor-adminserver:v1.7.5 "/harbor/start.sh" 6 weeks ago Up 6 weeks (healthy) harbor-adminserver
450ac364f681 goharbor/registry-photon:v2.6.2-v1.7.5 "/entrypoint.sh /etc…" 6 weeks ago Up 6 weeks (healthy) 5000/tcp registry
2dcd38beeb52 goharbor/redis-photon:v1.7.5 "docker-entrypoint.s…" 6 weeks ago Up 6 weeks 6379/tcp redis
21153d5f9cc1 goharbor/harbor-registryctl:v1.7.5 "/harbor/start.sh" 6 weeks ago Up 6 weeks (healthy) registryctl
cb21ef495dde goharbor/harbor-log:v1.7.5 "/bin/sh -c /usr/loc…" 6 weeks ago Up 6 weeks (healthy) 127.0.0.1:1514->10514/tcp harbor-log
[root@harbor25 certs]#
2.4 登录harbor
- 用户名 admin
- 密码 Harbor12345 (默认)
3.harbor高可用功能配置
- harbor.haostack.com和harbor25.haostack.com 实现高可用
3.1 高可用复制功能配置
3.1.1 仓库管理
- 创建仓库 baseimages
3.1.2 复制管理配置
- 同步baseimages这个项目
3.1.3 验证复制任务
3.1.4 在harbor25上验证，同步完成
3.1.5重新上传一个镜像完整步骤
- 镜像 centos:7.2.1511
//docker.io 上下载一个镜像
[root@harbor25 harbor]# docker pull centos:7.2.1511
7.2.1511: Pulling from library/centos
a8c7037c15e9: Pull complete
Digest: sha256:50cca1e74da4b6a4eb4ade029c8fdd4ee8564776801914d9bd89df8c6344add0
Status: Downloaded newer image for centos:7.2.1511
docker.io/library/centos:7.2.1511
//已经下载成功
[root@harbor25 harbor]# docker image ls
REPOSITORY TAG IMAGE ID CREATED SIZE
harbor.haostack.com/baseimages/nginx v1 4392e5dad77d 40 hours ago 132MB
goharbor/chartmuseum-photon v0.8.1-v1.7.5 ca910ae935e8 14 months ago 113MB
goharbor/harbor-migrator v1.7.5 326685e45678 14 months ago 679MB
goharbor/redis-photon v1.7.5 6886d1b5b009 14 months ago 101MB
goharbor/clair-photon v2.0.8-v1.7.5 970bdadc8995 14 months ago 164MB
goharbor/notary-server-photon v0.6.1-v1.7.5 667b2f0fb68d 14 months ago 135MB
goharbor/notary-signer-photon v0.6.1-v1.7.5 9ea20ae211bc 14 months ago 132MB
goharbor/harbor-registryctl v1.7.5 b5781fe85ba3 14 months ago 102MB
goharbor/registry-photon v2.6.2-v1.7.5 789d7a921303 14 months ago 86.7MB
goharbor/nginx-photon v1.7.5 172196970055 14 months ago 35.9MB
goharbor/harbor-log v1.7.5 26784a32e151 14 months ago 81.4MB
goharbor/harbor-jobservice v1.7.5 0755114a9c68 14 months ago 84.1MB
goharbor/harbor-core v1.7.5 86925c0ee5d5 14 months ago 95.6MB
goharbor/harbor-portal v1.7.5 9a9082e62030 14 months ago 40.6MB
goharbor/harbor-adminserver v1.7.5 b2afe190593f 14 months ago 72.3MB
goharbor/harbor-db v1.7.5 c1226370ffaa 14 months ago 138MB
centos 7.2.1511 9aec5c5fe4ba 14 months ago 195MB
//给镜像打tag
[root@harbor25 harbor]# docker login harbor.haostack.com
Authenticating with existing credentials...
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
//上传镜像
[root@harbor25 harbor]# docker push harbor.haostack.com/baseimages/centos:7.2.1511
The push refers to repository [harbor.haostack.com/baseimages/centos]
a11c91bfd866: Pushed
7.2.1511: digest: sha256:4516078f48ca34907ed8f874005473059dc29d46745d139c7cea85e3d474f4fb size: 529
[root@harbor25 harbor]#
3.1.6 验证复制功能
- 需要在harbor25上验证，已经同步
4.harbor双向同步
4.1在harbor26主机上配置
![[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-Gbsk7PeP-1595202143212)(60A9FA7CF6D149DEA9A50B4438BD55DE)]](https://img-blog.csdnimg.cn/20200720074628199.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NTA4MjY4Mw==,size_16,color_FFFFFF,t_70)
4.2在harbor25主机上配置
![[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-2aYOlSXx-1595202143213)(475625A3DABB42FEB3FA3F500B48ECAA)]](https://img-blog.csdnimg.cn/2020072007464372.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80NTA4MjY4Mw==,size_16,color_FFFFFF,t_70)
5.配置https证书
5.1 harbor25主机上配置https证书
5.1.1 创建证书目录
/usr/local/src/harbor/certs
5.1.2.创建CA证书
[root@harbor25 certs]# openssl req -newkey rsa:4096 -nodes -sha256 -keyout harbor-ca.key -x509 -days 365 -out harbor-ca.crt
Generating a 4096 bit RSA private key
.........++
..................................................++
writing new private key to 'harbor-ca.key'
-----
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:SH
Locality Name (eg, city) [Default City]:SH
Organization Name (eg, company) [Default Company Ltd]:YV
Organizational Unit Name (eg, section) []:YV
Common Name (eg, your name or your server's hostname) []:harbor25.haostack.com
Email Address []:
//生成的文件
[root@harbor25 certs]# ll
total 8
-rw-r--r-- 1 root root 1996 Jun 4 18:39 harbor-ca.crt
-rw-r--r-- 1 root root 3272 Jun 4 18:39 harbor-ca.key
5.1.3 生成CA证书签名请求
[root@harbor25 certs]# openssl req -newkey rsa:4096 -nodes -sha256 -keyout harbor25.haostack.com.key -out harbor25.haostack.com.csr
Generating a 4096 bit RSA private key
.....................................................................................................................................
........................++
writing new private key to 'harbor25.haostack.com.key'
-----
You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:SH
Locality Name (eg, city) [Default City]:SH
Organization Name (eg, company) [Default Company Ltd]:YV
Organizational Unit Name (eg, section) []:YV
Common Name (eg, your name or your server's hostname) []:harbor25.haostack.com
Email Address []:
Please enter the following 'extra' attributes
to be sent with your certificate request
A challenge password []:
An optional company name []:
//生成了两个签名文件 harbor25.haostack.com.csr harbor25.haostack.com.key
[root@harbor25 certs]# ll
total 16
-rw-r--r-- 1 root root 1687 Jun 4 18:40 harbor25.haostack.com.csr
-rw-r--r-- 1 root root 3272 Jun 4 18:40 harbor25.haostack.com.key
-rw-r--r-- 1 root root 1996 Jun 4 18:39 harbor-ca.crt
-rw-r--r-- 1 root root 3272 Jun 4 18:39 harbor-ca.key
5.1.4 生成主机证书
[root@harbor25 certs]# openssl x509 -req -days 365 -in harbor25.haostack.com.csr -CA harbor-ca.crt -CAkey harbor-ca.key -CAcreateseri
Signature ok
subject=/C=CN/ST=SH/L=SH/O=YV/OU=YV/CN=harbor25.haostack.com
Getting CA Private Key
// 根据ca证书文件，生成了两个主机证书文件
harbor25.haostack.com.crt
harbor-ca.srl
[root@harbor25 certs]# ll
total 24
-rw-r--r-- 1 root root 1879 Jun 4 18:40 harbor25.haostack.com.crt
-rw-r--r-- 1 root root 1687 Jun 4 18:40 harbor25.haostack.com.csr
-rw-r--r-- 1 root root 3272 Jun 4 18:40 harbor25.haostack.com.key
-rw-r--r-- 1 root root 1996 Jun 4 18:39 harbor-ca.crt
-rw-r--r-- 1 root root 3272 Jun 4 18:39 harbor-ca.key
-rw-r--r-- 1 root root 17 Jun 4 18:40 harbor-ca.srl
5.1.5 配置harbor.cfg
[root@harbor25 harbor]# more harbor.cfg
## Configuration file of Harbor
#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
_version = 1.7.0
#The IP address or hostname to access admin UI and registry service.
#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail.
hostname = harbor25.haostack.com(修改为域名)
#The protocol for accessing the UI and token/notification service, by default it is http.
#It can be set to https if ssl is enabled on nginx.
ui_url_protocol = https (修改为https)
#Maximum number of job workers in job service
max_job_workers = 10
#Determine whether or not to generate certificate for the registry's token.
#If the value is on, the prepare script creates new root cert and private key
#for generating token to access the registry. If the value is off the default key/cert will be used.
#This flag also controls the creation of the notary signer's cert.
customize_crt = on
#The path of cert and key files for nginx, they are applied only the protocol is set to https
//证书目录按照要求更改
ssl_cert = /usr/local/src/harbor/certs/harbor-ca.crt
ssl_cert_key =/usr/local/src/harbor/certs/harbor-ca.key
#The path of secretkey storage
secretkey_path = /usr/local/src/harbor/certs
5.1.6 重新配置harbor文件
./prepare
docker-compose down
docker-compose up -d
5.1 7.客户端测试
- 每一个客户端都需要复制上面的harbor-ca.crt到Docker相应目录，然后重启Docker
#客户端创建证书目录
/etc/docker/certs.d/harbor25.haostack.com
//需要把之前制作的ca证书添加到信任（因为是自签名证书）：传到客户端证书目录
[root@harbor25 harbor25.haostack.com]# ll
total 4
-rw-r--r-- 1 root root 1996 Jun 4 18:46 harbor-ca.crt
[root@harbor25 harbor25.haostack.com]# pwd
/etc/docker/certs.d/harbor25.haostack.com
[root@harbor25 harbor25.haostack.com]#
//重启服务
systemctl restart docker.service
5.1 7.1 linux客户端测试
root@harbor26 harbor25.haostack.com]# docker login harbor25.haostack.com
Username: admin
Password:
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@harbor26 harbor25.haostack.com]#
5.1 7.2 Windows客户端测试
5.2.harbor26主机配置https证书
5.2.1 把harbor-ca.crt拷贝到客户端中
- 原理一模一样
[root@harbor26 certs]# scp harbor-ca.crt root@172.16.62.25:/etc/docker/certs.d/harbor.haostack.com/
root@172.16.62.25's password:
harbor-ca.crt 100% 1992 1.1MB/s 00:00
[root@harbor26 certs]#
[root@harbor26 certs]#
5.2.2 Linux客户端测试
[root@harbor25 harbor.haostack.com]# docker login harbor.haostack.com
Authenticating with existing credentials...
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@harbor25 harbor.haostack.com]# pwd
/etc/docker/certs.d/harbor.haostack.com
5.2.3 Windows客户端测试
5.3 客户端配置私有仓库步骤
5.3.1 客户端DNS需要指向172.16.62.24
#客户端DNS需要指向172.16.62.24
[root@docker28 mysql]# cat /etc/resolv.conf
nameserver 172.16.62.24
5.3.2 需要把在客户端配置文件中指向仓库地址
- 仓库地址 harbor.haostack.com
#目录 /usr/lib/systemd/system/docker.service
#需要添加 --containerd=/run/containerd/containerd.sock
[root@docker28 mysql]# more /usr/lib/systemd/system/docker.service | grep ExecStart
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --insecure-registry=harbor.haostack.com --bip=172.18.0.1/16
[root@docker28 mysql]
5.3.3 重启服务
[root@docker28 mysql]# systemctl daemon-reload
[root@docker28 mysql]# systemctl restart docker
5.3.4 登录harbor仓库
[root@docker28 mysql]# docker login harbor.haostack.com
Username: admin
Password: #Harbor12345（默认）
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
[root@docker28 mysql]#
5.3.5 下载镜像
[root@docker28 mysql]# docker pull harbor.haostack.com/baseimages/jack_tomcat_app2:v1
v1: Pulling from baseimages/jack_tomcat_app2
9b4ebb48de8d: Pull complete
d924bed05a91: Pull complete
3f3d7e0bdf4e: Pull complete
3978de2fe740: Pull complete
d0c90f05d39d: Pull complete
90c46cb81e22: Pull complete
02d52509eea0: Pull complete
8c980781151a: Pull complete
4b67d92370c0: Pull complete
de10019bb4af: Pull complete
236d2fad78b8: Pull complete
Digest: sha256:638a69df1f124290d8da7439deef987a9108409da925585821f67a728a144e8a
Status: Downloaded newer image for harbor.haostack.com/baseimages/jack_tomcat_app2:v1
harbor.haostack.com/baseimages/jack_tomcat_app2:v1
5.3.6 查看镜像
[root@docker28 mysql]# docker image ls
REPOSITORY TAG IMAGE ID CREATED SIZE
harbor.haostack.com/baseimages/jack_tomcat_app2 v1 a716733c7cfa About an hour ago 951MB
busybox latest c7c37e472d31 2 weeks ago 1.22MB
[root@docker28 mysql]#
2.单机编排docker-compose
1.docker-compose安装
#安装
[root@docker27 haproxy]# yum install docker-compose -y
#docker-compose帮助
[root@docker27 haproxy]# docker-compose --help
Define and run multi-container applications with Docker.
Usage:
docker-compose [-f <arg>...] [options] [COMMAND] [ARGS...]
docker-compose -h|--help
Options:
-f, --file FILE Specify an alternate compose file (default: docker-compose.yml)
-p, --project-name NAME Specify an alternate project name (default: directory name)
--verbose Show more output
--no-ansi Do not print ANSI control characters
-v, --version Print version and exit
-H, --host HOST Daemon socket to connect to
--tls Use TLS; implied by --tlsverify
--tlscacert CA_PATH Trust certs signed only by this CA
--tlscert CLIENT_CERT_PATH Path to TLS certificate file
--tlskey TLS_KEY_PATH Path to TLS key file
--tlsverify Use TLS and verify the remote
--skip-hostname-check Don't check the daemon's hostname against the name specified
in the client certificate (for example if your docker host
is an IP address)
--project-directory PATH Specify an alternate working directory
(default: the path of the Compose file)
Commands:
build Build or rebuild services
bundle Generate a Docker bundle from the Compose file
config Validate and view the Compose file
create Create services
down Stop and remove containers, networks, images, and volumes
events Receive real time events from containers
exec Execute a command in a running container
help Get help on a command
images List images
kill Kill containers
logs View output from containers
pause Pause services
port Print the public port for a port binding
ps List containers
pull Pull service images
push Push service images
restart Restart services
rm Remove stopped containers
run Run a one-off command
scale Set number of containers for a service
start Start services
stop Stop services
top Display the running processes
unpause Unpause services
up Create and start containers
version Show the Docker-Compose version information
#查看docker-compose版本
[root@docker27 haproxy_nginx_tomcat]# docker-compose version
docker-compose version 1.18.0, build 8dd22a9
docker-py version: 2.6.1
CPython version: 3.6.8
OpenSSL version: OpenSSL 1.0.2k-fips 26 Jan 2017
[root@docker27 haproxy_nginx_tomcat]#
2.环境准备
- 从harbor中下载镜像
#镜像下载完成
harbor.haostack.com/baseimages/jack_nginx_yum v1 13cc2282b9fe 2 days ago 677MB
harbor.haostack.com/baseimages/jack_tomcat_app3 v1 34da15a067b6 14 hours ago 951MB
harbor.haostack.com/baseimages/jack_tomcat_app2 v1 a716733c7cfa 14 hours ago 951MB
harbor.haostack.com/baseimages/jack_tomcat_app1 v1 a52bf7705749 37 hours ago 951MB
harbor.haostack.com/baseimages/jack_centos_haproxy v2 b7944130b017 7 hours ago 572MB
#文件准备
[root@docker27 haproxy_nginx_tomcat]# ll
total 20
-rwxr-xr-x 1 root root 61 Jul 19 18:06 build-command.sh
-rw-r--r-- 1 root root 1225 Jul 20 01:13 docker-compose.yml
-rw-r--r-- 1 root root 1148 Jul 20 00:30 haproxy.cfg
drwxr-xr-x 2 nginx nginx 46 Jul 19 22:07 log
-rw-r--r-- 1 root root 2012 Jul 20 00:39 nginx.conf
drwxr-xr-x 2 root root 24 Jul 19 19:10 webpage
[root@docker27 haproxy_nginx_tomcat]#
3.通过docker-compose 创建容器
3.1 docker-compose.yml配置
[root@docker27 haproxy_nginx_tomcat]# more docker-compose.yml
haproxy:
image: harbor.haostack.com/baseimages/jack_centos_haproxy:v2
container_name: haproxy
expose:
- 80
- 9999
ports:
- "80:80"
- "9999:9999"
volumes:
- /data/docker-compose/haproxy_nginx_tomcat/haproxy.cfg:/etc/haproxy/haproxy.cfg #卷映射
links:
- nginx-web1 #链接到nginx-web1容器上
nginx-web1:
image: harbor.haostack.com/baseimages/jack_nginx_yum:v1
container_name: nginx-web1 #容器名称
ports:
- "8801:80"
- "443:443"
volumes:
- /data/docker-compose/haproxy_nginx_tomcat/nginx.conf:/etc/nginx/nginx.conf #卷映射nginx.conf
- /data/docker-compose/haproxy_nginx_tomcat/log:/var/log/nginx #卷映射log目录
- /data/docker-compose/haproxy_nginx_tomcat/webpage:/usr/share/nginx/html/webpage #卷映射webpage
links:
- tomcat-app1 #链接到tomcat-app1容器上
- tomcat-app2 #链接到tomcat-app2容器上
- tomcat-app3 #链接到tomcat-app3容器上
tomcat-app1:
image: harbor.haostack.com/baseimages/jack_tomcat_app1:v1
container_name: tomcat-app1
expose:
- 8080
ports:
- "8081:8080"
tomcat-app2:
image: harbor.haostack.com/baseimages/jack_tomcat_app2:v1
container_name: tomcat-app2
expose:
- 8080
ports:
- "8082:8080"
tomcat-app3:
image: harbor.haostack.com/baseimages/jack_tomcat_app3:v1
container_name: tomcat-app3
expose:
- 8080
ports:
- "8083:8080"
[root@docker27 haproxy_nginx_tomcat]#
3.2 haproxy配置
[root@docker27 haproxy_nginx_tomcat]# more haproxy.cfg
global
maxconn 100000
chroot /apps/haproxy
stats socket /apps/haproxy/haproxy.sock mode 600 level admin
uid 99
gid 99
daemon
pidfile /apps/haproxy/run/haproxy.pid
log 127.0.0.1 local3 info
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
listen stats
bind 0.0.0.0:9999
stats enable
stats hide-version
stats uri /haproxy-status
stats realm HAPorxy\Stats\Page
stats auth haadmin:123456
stats refresh 3s
stats admin if TRUE
mode http
log global
listen nginx-web
bind 0.0.0.0:80
mode http
log global
server web1 nginx-web1:80 check inter 3000 fall 3 rise 5 #需要指向nginx-web1容器
[root@docker27 haproxy_nginx_tomcat]#
3.3 nginx配置
[root@docker27 haproxy_nginx_tomcat]# more nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
daemon off;
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
http {
log_format my_access_json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"uri":"$uri",'
'"domain":"$host",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"tcp_xff":"$proxy_protocol_addr",'
'"http_user_agent":"$http_user_agent",'
'"status":"$status"}';
access_log /var/log/nginx/access_json.log my_access_json;
sendfile on;
keepalive_timeout 65;
include mime.types;
default_type text/html;
charset utf-8;
upstream tomcat_server {
server tomcat-app1:8080;
server tomcat-app2:8080;
server tomcat-app3:8080;
}
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name _;
root /usr/share/nginx/html;
include /etc/nginx/default.d/*.conf;
location / {
}
location /app1 {
proxy_pass http://tomcat_server;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
}
error_page 404 /404.html;
location = /40x.html {
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
}
}
}
[root@docker27 haproxy_nginx_tomcat]#
4.创建容器
Creating tomcat-app3 ... done
Creating nginx-web1 ... done
Creating haproxy ... done
Creating tomcat-app1 ...
Creating tomcat-app3 ...
Creating nginx-web1 ...
Creating haproxy ...
[root@docker27 haproxy_nginx_tomcat]#
5. 查看容器
[root@docker27 haproxy_nginx_tomcat]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
e5941cb0731c harbor.haostack.com/baseimages/jack_centos_haproxy:v2 "/usr/bin/run_haprox…" 54 seconds ago Up 53 seconds 0.0.0.0:80->80/tcp, 0.0.0.0:9999->9999/tcp haproxy
aa023f5ae78c harbor.haostack.com/baseimages/jack_nginx_yum:v1 "nginx" 55 seconds ago Up 54 seconds 0.0.0.0:443->443/tcp, 0.0.0.0:8801->80/tcp nginx-web1
1bac9ffa948c harbor.haostack.com/baseimages/jack_tomcat_app3:v1 "/apps/tomcat/bin/ru…" 56 seconds ago Up 55 seconds 8443/tcp, 0.0.0.0:8083->8080/tcp tomcat-app3
1a0276fc5173 harbor.haostack.com/baseimages/jack_tomcat_app1:v1 "/apps/tomcat/bin/ru…" 56 seconds ago Up 55 seconds 8443/tcp, 0.0.0.0:8081->8080/tcp tomcat-app1
23eba27d2fbc harbor.haostack.com/baseimages/jack_tomcat_app2:v1 "/apps/tomcat/bin/ru…" 56 seconds ago Up 55 seconds 8443/tcp, 0.0.0.0:8082->8080/tcp tomcat-app2
[root@docker27 haproxy_nginx_tomcat]#
6.测试
6.1webpage页面测试
[root@85383266b12f nginx]# curl http://172.16.62.27/webpage/
docker nginx test
6.2 nginx静态页面测试
[root@85383266b12f nginx]# curl http://172.16.62.27
docker nginx v1
[root@85383266b12f nginx]#
6.3通过访问nginx转发到tomcat上
[root@85383266b12f nginx]# curl http://172.16.62.27/app1/
tomcat app1 web server
docker
780
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
