openstack-helm

最新推荐文章于 2024-08-23 09:14:38 发布

wen328

最新推荐文章于 2024-08-23 09:14:38 发布

阅读量1.7k

点赞数 1

分类专栏： k8s 文章标签： kubernetes openstack

本文链接：https://blog.csdn.net/wen328/article/details/119759274

版权

k8s 专栏收录该内容

2 篇文章 1 订阅

订阅专栏

文章目录

openstack-helm 安装

openstack-helm 安装

helm有2个版本，分别为helm2和helm3，目前openstack-helm只支持helm2，因此：下载得helm版本为helm-v2.17.0

wget https://get.helm.sh/helm-v2.17.0-linux-amd64.tar.gz
tar -xvf helm-v2.17.0-linux-amd64.tar.gz
cp linux-amd64/helm /usr/local/bin/

helm

helm 初始化

# 要安装 Helm 的服务端程序，我们需要使用到kubectl工具，所以先确保kubectl工具能够正常的访问 kubernetes 集群的apiserver哦
# 初始化helm仓库
helm init --upgrade -i registry.cn-hangzhou.aliyuncs.com/google_containers/tiller:v2.17.0 --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts
helm serve &

# 另外一个值得注意的问题是RBAC，我们的 kubernetes 集群是1.8.x版本的，默认开启了RBAC访问控制，所以我们需要为Tiller创建一个ServiceAccount，让他拥有执行的权限，详细内容可以查看 Helm 文档中的Role-based Access Control。 创建rbac.yaml文件创建helm集群账户和角色
vim service-accounts.yaml
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: tiller
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: tiller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
  - kind: ServiceAccount
    name: tiller
    namespace: kube-system

kubectl create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default

openstack-helm下载

cd /opt
git clone https://github.com/openstack/openstack-helm.git
git clone https://github.com/openstack/openstack-helm-infra.git

openstack组件安装

安装依赖包
yum -y install make jq git curl

修改openstack安装版本

cd /opt/openstack-helm
git diff tools/deployment/common/setup-client.sh
-  -c${UPPER_CONSTRAINTS_FILE:=https://releases.openstack.org/constraints/upper/${OPENSTACK_RELEASE:-stein}} \
+  -c${UPPER_CONSTRAINTS_FILE:=https://releases.openstack.org/constraints/upper/${OPENSTACK_RELEASE:-ussuri}} \
 
 
git diff tools/deployment/common/get-values-overrides.sh
-: "${OPENSTACK_RELEASE:="train"}"
+: "${OPENSTACK_RELEASE:="ussuri"}"

创建OpenStack clients 和 Kubernetes RBAC rules

./tools/deployment/developer/common/020-setup-client.sh

helm再部署过程中，会依赖nodes label进行调度，因此需要给节点添加标签

kubectl label nodes 10.2.11.176 nginx-ingress=enabled
kubectl label nodes 10.2.11.177 nginx-ingress=enabled
kubectl label nodes 10.2.11.176 openstack-control-plane=enabled
kubectl label nodes 10.2.11.177 openstack-control-plane=enabled
kubectl label nodes 10.2.11.178 openstack-control-plane=enabled
kubectl label nodes 10.2.11.176 openstack-compute-node=enabled
kubectl label nodes 10.2.11.177 openstack-compute-node=enabled
kubectl label nodes 10.2.11.178 openstack-compute-node=enabled
kubectl label nodes 10.2.11.176 openvswitch=enabled
kubectl label nodes 10.2.11.177 openvswitch=enabled
kubectl label nodes 10.2.11.178 openvswitch=enabled

安装ingress

# 修改ingress镜像，原始镜像被墙了，下载不下来
cd /opt/openstack-helm-infra/
git diff ingress/values.yaml
-    ingress: k8s.gcr.io/ingress-nginx/controller:v0.42.0
-    ingress_module_init: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
-    ingress_routed_vip: docker.io/openstackhelm/neutron:stein-ubuntu_bionic
-    error_pages: k8s.gcr.io/defaultbackend:1.4
+    ingress: docker.io/willdockerhub/ingress-nginx-controller:v0.42.0
+    ingress_module_init: docker.io/openstackhelm/neutron:ussuri-ubuntu_bionic
+    ingress_routed_vip: docker.io/openstackhelm/neutron:ussuri-ubuntu_bionic
+    error_pages: docker.io/chenliujin/defaultbackend:1.4
 labels:
   server:
-    node_selector_key: openstack-control-plane
+    node_selector_key: nginx-ingress
     node_selector_value: enabled
   error_server:
-    node_selector_key: openstack-control-plane
+    node_selector_key: nginx-ingress
     node_selector_value: enabled
-    addr: 172.18.0.1/11
+    addr: 10.221.0.1/11

执行部署脚本

OSH_DEPLOY_MULTINODE=True ./tools/deployment/component/common/ingress.sh

安装ceph

修改openstack-helm ceph脚本

git diff tools/deployment/multinode/030-ceph.sh
diff --git a/tools/deployment/multinode/030-ceph.sh b/tools/deployment/multinode/030-ceph.sh
index b3fa8db2..f7cc9ed8 100755
--- a/tools/deployment/multinode/030-ceph.sh
+++ b/tools/deployment/multinode/030-ceph.sh
@@ -37,13 +37,13 @@ network:
   cluster: ${CEPH_CLUSTER_NETWORK}
 deployment:
   storage_secrets: true
-  ceph: true
-  rbd_provisioner: true
-  csi_rbd_provisioner: true
+  ceph: false
+  rbd_provisioner: false
+  csi_rbd_provisioner: false
   cephfs_provisioner: false
   client_secrets: false
 bootstrap:
-  enabled: true
+  enabled: false
 conf:
   ceph:
     global:
@@ -74,7 +74,7 @@ manifests:
 EOF
 
 : ${OSH_INFRA_PATH:="../openstack-helm-infra"}
-for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do
+for CHART in ceph-mon ; do
   make -C ${OSH_INFRA_PATH} ${CHART}
   helm upgrade --install ${CHART} ${OSH_INFRA_PATH}/${CHART} \
     --namespace=ceph \
@@ -82,14 +82,4 @@ for CHART in ceph-mon ceph-osd ceph-client ceph-provisioners; do
     ${OSH_EXTRA_HELM_ARGS} \
     ${OSH_EXTRA_HELM_ARGS_CEPH_DEPLOY}
 
-  #NOTE: Wait for deploy
-  ./tools/deployment/common/wait-for-pods.sh ceph 1200
-
-  #NOTE: Validate deploy
-  MON_POD=$(kubectl get pods \
-    --namespace=ceph \
-    --selector="application=ceph" \
-    --selector="component=mon" \
-    --no-headers | awk '{ print $1; exit }')
-  kubectl exec -n ceph ${MON_POD} -- ceph -s
 done
 
git diff tools/deployment/multinode/kube-node-subnet.sh
diff --git a/tools/deployment/multinode/kube-node-subnet.sh b/tools/deployment/multinode/kube-node-subnet.sh
index 08f069a8..9ed56742 100755
--- a/tools/deployment/multinode/kube-node-subnet.sh
+++ b/tools/deployment/multinode/kube-node-subnet.sh
@@ -19,7 +19,6 @@ kubectl get nodes -o json | jq -r '.items[].status.addresses[] | select(.type=="
 function run_and_log_ipcalc {
   POD_NAME="tmp-$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-z | head -c 5; echo)"
   kubectl run ${POD_NAME} \
-    --generator=run-pod/v1 \
     --wait \
     --image ${UTILS_IMAGE} \
     --restart=Never \

修改openstack-helm-infra ceph镜像名称

git diff ceph-client/values.yaml
diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml
index 92c31611..a6920dcb 100644
--- a/ceph-client/values.yaml
+++ b/ceph-client/values.yaml
@@ -24,11 +24,11 @@ release_group: null
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
-    ceph_mds: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
-    ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
-    ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
+    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
+    ceph_mds: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
+    ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
+    ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/library/docker:17.07.0'
 
git diff ceph-mon/values.yaml
diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml
index f060c13a..7d284d67 100644
--- a/ceph-mon/values.yaml
+++ b/ceph-mon/values.yaml
@@ -23,10 +23,10 @@ deployment:
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
-    ceph_mon: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
-    ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
+    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
+    ceph_mon: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
+    ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/library/docker:17.07.0'
 
git diff ceph-osd/values.yaml
diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml
index 7277a73c..906810e3 100644
--- a/ceph-osd/values.yaml
+++ b/ceph-osd/values.yaml
@@ -19,9 +19,9 @@
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_osd: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
-    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
+    ceph_osd: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
+    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/library/docker:17.07.0'
 
git diff ceph-provisioners/values.yaml
diff --git a/ceph-provisioners/values.yaml b/ceph-provisioners/values.yaml
index b4ab0a9d..a6f9c0b7 100644
--- a/ceph-provisioners/values.yaml
+++ b/ceph-provisioners/values.yaml
@@ -30,10 +30,10 @@ release_group: null
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
+    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
     ceph_cephfs_provisioner: 'docker.io/openstackhelm/ceph-cephfs-provisioner:ubuntu_bionic-20200521'
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
-    ceph_rbd_provisioner: 'docker.io/openstackhelm/ceph-rbd-provisioner:change_770201_ubuntu_bionic-20210113'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
+    ceph_rbd_provisioner: 'docker.io/openstackhelm/ceph-rbd-provisioner:latest-ubuntu_bionic'
     csi_provisioner: 'quay.io/k8scsi/csi-provisioner:v1.6.0'
     csi_snapshotter: 'quay.io/k8scsi/csi-snapshotter:v2.1.1'
     csi_attacher: 'quay.io/k8scsi/csi-attacher:v2.1.1'
 
git diff ceph-rgw/values.yaml
diff --git a/ceph-rgw/values.yaml b/ceph-rgw/values.yaml
index a5147856..21aedfec 100644
--- a/ceph-rgw/values.yaml
+++ b/ceph-rgw/values.yaml
@@ -24,13 +24,13 @@ release_group: null
 images:
   pull_policy: IfNotPresent
   tags:
-    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
-    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
-    ceph_rgw: 'docker.io/openstackhelm/ceph-daemon:change_770201_ubuntu_bionic-20210113'
+    ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
+    ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
+    ceph_rgw: 'docker.io/openstackhelm/ceph-daemon:latest-ubuntu_bionic'
     dep_check: 'quay.io/airshipit/kubernetes-entrypoint:v1.0.0'
     image_repo_sync: 'docker.io/library/docker:17.07.0'
-    rgw_s3_admin: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
-    rgw_placement_targets: 'docker.io/openstackhelm/ceph-config-helper:change_770201_ubuntu_bionic-20210113'
+    rgw_s3_admin: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
+    rgw_placement_targets: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_bionic'
     ks_endpoints: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
     ks_service: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'
     ks_user: 'docker.io/openstackhelm/heat:newton-ubuntu_xenial'

修改ceph-mon模板文件

git diff ceph-mon/templates/bin/mon/_start.sh.tpl
diff --git a/ceph-mon/templates/bin/mon/_start.sh.tpl b/ceph-mon/templates/bin/mon/_start.sh.tpl
index b045a39e..96745c2b 100644
--- a/ceph-mon/templates/bin/mon/_start.sh.tpl
+++ b/ceph-mon/templates/bin/mon/_start.sh.tpl
@@ -28,49 +28,6 @@ if [[ -z "$CEPH_PUBLIC_NETWORK" ]]; then
   exit 1
 fi
 
-if [[ -z "$MON_IP" ]]; then
-  echo "ERROR- MON_IP must be defined as the IP address of the monitor"
-  exit 1
-fi
-
-if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then
-    MON_NAME=${POD_NAME}
-else
-    MON_NAME=${NODE_NAME}
-fi
-MON_DATA_DIR="/var/lib/ceph/mon/${CLUSTER}-${MON_NAME}"
-MONMAP="/etc/ceph/monmap-${CLUSTER}"
-
-# Make the monitor directory
-/bin/sh -c "mkdir -p \"${MON_DATA_DIR}\""
-
-function get_mon_config {
-  # Get fsid from ceph.conf
-  local fsid=$(ceph-conf --lookup fsid -c /etc/ceph/${CLUSTER}.conf)
-
-  timeout=10
-  MONMAP_ADD=""
-
-  while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do
-    # Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params
-    if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then
-        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--addv {{`{{.metadata.name}}`}} [v1:{{`{{.status.podIP}}`}}:${MON_PORT},v2:{{`{{.status.podIP}}`}}:${MON_PORT_V2}] {{`{{end}}`}} {{`{{end}}`}}")
-    else
-        MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--addv {{`{{.spec.nodeName}}`}} [v1:{{`{{.status.podIP}}`}}:${MON_PORT},v2:{{`{{.status.podIP}}`}}:${MON_PORT_V2}] {{`{{end}}`}} {{`{{end}}`}}")
-    fi
-    (( timeout-- ))
-    sleep 1
-  done
-
-  if [[ -z "${MONMAP_ADD// }" ]]; then
-      exit 1
-  fi
-
-  # Create a monmap with the Pod Names and IP
-  monmaptool --create ${MONMAP_ADD} --fsid ${fsid} ${MONMAP} --clobber
-}
-
-get_mon_config
 
 # If we don't have a monitor keyring, this is a new monitor
 if [ ! -e "${MON_DATA_DIR}/keyring" ]; then
@@ -81,33 +38,9 @@ if [ ! -e "${MON_DATA_DIR}/keyring" ]; then
     cp -vf ${MON_KEYRING}.seed ${MON_KEYRING}
   fi
 
-  if [ ! -e ${MONMAP} ]; then
-    echo "ERROR- ${MONMAP} must exist. You can extract it from your current monitor by running 'ceph mon getmap -o ${MONMAP}' or use a KV Store"
-    exit 1
-  fi
-
   # Testing if it's not the first monitor, if one key doesn't exist we assume none of them exist
   for KEYRING in ${OSD_BOOTSTRAP_KEYRING} ${MDS_BOOTSTRAP_KEYRING} ${ADMIN_KEYRING}; do
     ceph-authtool ${MON_KEYRING} --import-keyring ${KEYRING}
   done
-
-  # Prepare the monitor daemon's directory with the map and keyring
-  ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" --mkfs -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}"
-else
-  echo "Trying to get the most recent monmap..."
-  # Ignore when we timeout, in most cases that means the cluster has no quorum or
-  # no mons are up and running yet
-  timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true
-  ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}"
-  timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT_V2}" || true
 fi
 
-# start MON
-exec /usr/bin/ceph-mon \
-  --cluster "${CLUSTER}" \
-  --setuser "ceph" \
-  --setgroup "ceph" \
-  -d \
-  -i ${MON_NAME} \
-  --mon-data "${MON_DATA_DIR}" \
-  --public-addr "${MON_IP}:${MON_PORT_V2}"

修改ceph-client-admin-keyring secrets
将ceph admin得密钥镜像base64编码，修改ceph.client.admin.keyring

kubectl edit ceph-client-admin-keyring -n ceph

apiVersion: v1
data:
  keyring: Y2xpZW50LmFkbWluCiAgICAgICAga2V5OiBBUUFoZXR0ZEFBQUFBQkFBRUVSL3F1S0p5bGhFRDZ2UXg3YzhWdz09CiAgICAgICAgY2FwczogW21kc10gYWxsb3cgKgogICAgICAgIGNhcHM6IFttZ3JdIGFsbG93ICoKICAgICAgICBjYXBzOiBbbW9uXSBhbGxvdyAqCiAgICAgICAgY2FwczogW29zZF0gYWxsb3cgKg==
kind: Secret
metadata:
  name: ceph-client-admin-keyring
  namespace: ceph

tee /tmp/pvc-ceph-client-key.yaml <<EOF
apiVersion: v1
data:
  key: QVFBaGV0dGRBQUFBQUJBQUVFUi9xdUtKeWxoRUQ2dlF4N2M4Vnc9PQ==
kind: Secret
metadata:
  labels:
    application: ceph
    component: rbd
    release_group: ceph-openstack-config
  name: pvc-ceph-client-key
  namespace: openstack
type: kubernetes.io/rbd
EOF

kubectl apply -f /tmp/pvc-ceph-client-key.yaml

执行部署脚本

# 1.nodes节点打标签 2.修改ceph相关/values.yaml文件，镜像名称 3.创建ceph-mon-discovery endpoints
./tools/deployment/multinode/030-ceph.sh

openstack-ceph

创建ceph-mon-discovery Endpoints

tee /tmp/ceph-mon-discovery.yaml <<EOF
apiVersion: v1
kind: Endpoints
metadata:
  labels:
    app: ceph
    mon_cluster: ceph
    rook_cluster: ceph
  name: ceph-mon-discovery
  namespace: ceph
subsets:
- addresses:
  - ip: 10.2.11.176
    nodeName: k8s-1
    targetRef:
      kind: Pod
      namespace: ceph
  - ip: 10.2.11.177
    nodeName: k8s-2
    targetRef:
      kind: Pod
      namespace: ceph
  - ip: 10.2.11.178
    nodeName: k8s-3
    targetRef:
      kind: Pod
      namespace: ceph

  ports:
  - name: mon-msgr2
    port: 3300
    protocol: TCP
  - name: mon
    port: 6789
    protocol: TCP
EOF

#配置ceph-openstack相关
./tools/deployment/multinode/040-ceph-ns-activate.sh

Mariadb

修改openstack-helm脚本，使mariadb使用k8s volume

git diff tools/deployment/multinode/050-mariadb.sh
diff --git a/tools/deployment/multinode/050-mariadb.sh b/tools/deployment/multinode/050-mariadb.sh
index 5ba6d44a..2d50dd14 100755
--- a/tools/deployment/multinode/050-mariadb.sh
+++ b/tools/deployment/multinode/050-mariadb.sh
@@ -11,8 +11,6 @@ make -C ${HELM_CHART_ROOT_PATH} mariadb
 : ${OSH_EXTRA_HELM_ARGS:=""}
 helm upgrade --install mariadb ${HELM_CHART_ROOT_PATH}/mariadb \
     --namespace=openstack \
-    --set volume.use_local_path_for_single_pod_cluster.enabled=true \
-    --set volume.enabled=false \
     --values=/tmp/mariadb.yaml \
     ${OSH_EXTRA_HELM_ARGS} \
     ${OSH_EXTRA_HELM_ARGS_MARIADB}

修改openstack-helm-infra脚本 mariadb镜像名称

git diff mariadb/values.yaml
diff --git a/mariadb/values.yaml b/mariadb/values.yaml
index dcc905dc..0885b266 100644
--- a/mariadb/values.yaml
+++ b/mariadb/values.yaml
@@ -21,8 +21,8 @@ release_group: null
 images:
   tags:
     mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal
-    ingress: k8s.gcr.io/ingress-nginx/controller:v0.42.0
-    error_pages: k8s.gcr.io/defaultbackend:1.4
+    ingress: docker.io/willdockerhub/ingress-nginx-controller:v0.42.0
+    error_pages: docker.io/chenliujin/defaultbackend:1.4
     prometheus_create_mysql_user: docker.io/library/mariadb:10.5.9-focal
     prometheus_mysql_exporter: docker.io/prom/mysqld-exporter:v0.12.1
     prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:newton-ubuntu_xenial

执行部署脚本

./tools/deployment/multinode/050-mariadb.sh

问题

# mariadb作为一个有状态服务，需要创建卷，因此，先创建ceph rbd块， 参考官网: https://docs.ceph.com/en/latest/rbd/rbd-kubernetes/
# 再helm安装得过程中，mariadb pod得状态是 没有ready得，这个问题解决参考如下：
kubectl get cm -n openstack
kubectl delete cm mariadb-mariadb-state -n openstack
helm delete mariadb --purge
# 然后再重新进行部署
./tools/deployment/multinode/050-mariadb.sh

RabbitMQ

执行安装脚本

./tools/deployment/multinode/060-rabbitmq.sh

问题解决：

vim /var/lib/kubelet/config.yaml

修改clusterDomain: cluster.local. 将后面得.去掉，然后重启kebelet服务

Memcached

执行安装脚本

./tools/deployment/multinode/070-memcached.sh

Keystone

执行安装脚本

./tools/deployment/multinode/080-keystone.sh

Glance

修改glance存储为rbd

git diff tools/deployment/multinode/100-glance.sh
diff --git a/tools/deployment/multinode/100-glance.sh b/tools/deployment/multinode/100-glance.sh
index 20bacd23..beba0683 100755
--- a/tools/deployment/multinode/100-glance.sh
+++ b/tools/deployment/multinode/100-glance.sh
@@ -23,7 +23,7 @@ make glance
 
 #NOTE: Deploy command
 : ${OSH_EXTRA_HELM_ARGS:=""}
-: ${GLANCE_BACKEND:="pvc"}
+: ${GLANCE_BACKEND:="rbd"}

执行安装脚本

./tools/deployment/multinode/100-glance.sh

Cinder

修改crush_rule

git diff tools/deployment/multinode/110-cinder.sh
diff --git a/tools/deployment/multinode/110-cinder.sh b/tools/deployment/multinode/110-cinder.sh
index 55f3af0a..a4136c6d 100755
--- a/tools/deployment/multinode/110-cinder.sh
+++ b/tools/deployment/multinode/110-cinder.sh
@@ -26,19 +26,19 @@ conf:
     pools:
       backup:
         replication: 1
-        crush_rule: same_host
+        crush_rule: replicated_rule
         chunk_size: 8
         app_name: cinder-backup
       # default pool used by rbd1 backend
       cinder.volumes:
         replication: 1
-        crush_rule: same_host
+        crush_rule: replicated_rule
         chunk_size: 8
         app_name: cinder-volume
       # secondary pool used by rbd2 backend
       cinder.volumes.gold:
         replication: 1
-        crush_rule: same_host
+        crush_rule: replicated_rule
         chunk_size: 8
         app_name: cinder-volume
   backends:

执行安装脚本

./tools/deployment/multinode/110-cinder.sh

OpenvSwitch

执行安装脚本

kubectl label nodes k8s-node-181 openvswitch=enabled
kubectl label nodes k8s-node-182 openvswitch=enabled
kubectl label nodes k8s-node-183 openvswitch=enabled

./tools/deployment/multinode/120-openvswitch.sh

Libvirt

执行安装脚本

# libvirt POD 为init状态，依赖neutron-ovs-agent pod
./tools/deployment/multinode/130-libvirt.sh

libvirt Pod启动完成之后，创建secret密钥, 此操作可以由Init容器初始化完成

# 原始容器中没有定义secret，再创建实例得时候会报错，手动进行创建。
virsh secret-list
 UUID                                  Usage
--------------------------------------------------------------------------------
# 进入容器中
kubectl exec -it libvirt-libvirt-default-2jwf6  -n openstack bash
生成密钥xml
cat << EOF > /etc/ceph.xml
<secret ephemeral="no" private="no">
<uuid>457eb676-33da-42ec-9a8c-9293d545c337</uuid>
<usage type="ceph">
<name>client.cinder. secret</name>
</usage>
</secret>
EOF
virsh secret-define --file /etc/ceph.xml
# secret UUID是定死得，base64是 ceph中client.cinder 得密钥
virsh secret-set-value --secret 457eb676-33da-42ec-9a8c-9293d545c337 --base64 AQAi3RRhsMJxGhAAWMZvqgo62ZZ4kUd30LvukA==
#重启libvirt容器

Compute Kit (Nova and Neutron)

修改配置

git diff tools/deployment/multinode/140-compute-kit.sh
diff --git a/tools/deployment/multinode/140-compute-kit.sh b/tools/deployment/multinode/140-compute-kit.sh
index 2fec7662..4d4367b7 100755
--- a/tools/deployment/multinode/140-compute-kit.sh
+++ b/tools/deployment/multinode/140-compute-kit.sh
@@ -16,7 +16,7 @@ set -xe
 : ${RUN_HELM_TESTS:="yes"}
 
 export OS_CLOUD=openstack_helm
-CEPH_ENABLED=false
+CEPH_ENABLED=true
 if openstack service list -f value -c Type | grep -q "^volume" && \
     openstack volume type list -f value -c Name | grep -q "rbd"; then
   CEPH_ENABLED=true
@@ -118,7 +118,7 @@ make neutron
 tee /tmp/neutron.yaml << EOF
 network:
   interface:
-    tunnel: docker0
+    tunnel: eth1

nova-bootstrap会下载外网镜像，提前准备好镜像，防止下载失败

wget http://10.2.11.2/dd_files/Kubernetes/hyperkube-amd64.tgz
docker load -i hyperkube-amd64.tgz

执行安装脚本

./tools/deployment/multinode/140-compute-kit.sh

安装得过程中，neutron可能安装失败，失败之后删掉重新跑

helm list
neutron                 1               Thu Aug 12 16:51:23 2021        FAILED          neutron-0.2.8                   v1.0.0          openstack
# 将charts删除
helm delete neutron --purge

重新执行

helm upgrade --install neutron ./neutron --namespace=openstack --values=/tmp/neutron.yaml --values=../openstack-helm/neutron/values_overrides/ussuri-ubuntu_bionic.yaml

Horizon

执行安装脚本

./tools/deployment/multinode/085-horizon.sh

dashboard安装完成之后，可能会造成资源访问不到得现象，解决如下:
安装traefik-ingress

wget http://10.2.11.2/dd_files/Kubernetes/traefik.tar.gz
tar -xvf traefik.tar.gz
cd traefik
# 创建资源
kubectl apply -f .
ingress.networking.k8s.io/horizon created
Warning: extensions/v1beta1 Ingress is deprecated in v1.14+, unavailable in v1.22+; use networking.k8s.io/v1 Ingress
ingress.extensions/horizon-a created
serviceaccount/traefik-ingress-controller created
deployment.apps/traefik-ingress-controller created
service/traefik-ingress-service created
clusterrole.rbac.authorization.k8s.io/traefik-ingress-controller created
clusterrolebinding.rbac.authorization.k8s.io/traefik-ingress-controller created
service/traefik-web-ui created
ingress.extensions/traefik-web-ui created
 
# 查看pod
kubectl get pod -n kube-system
traefik-ingress-controller-6dc7585cdc-97vhj   1/1     Running   0          2s
 
# 查看ingress
kubectl get ingress -n kube-system
NAME             CLASS    HOSTS                 ADDRESS   PORTS   AGE
traefik-web-ui   <none>   traefik-ui.minikube             80      2m7s

验证

查看ingress

kubectl get ingress -n openstack
NAME                          CLASS    HOSTS                                                                                               ADDRESS                               PORTS   AGE
cinder                      <none>   cinder,cinder.openstack,cinder.openstack.svc.cluster.local                                        10.2.11.177,10.2.11.178               80      50m
glance                      <none>   glance,glance.openstack,glance.openstack.svc.cluster.local                                        10.2.11.177,10.2.11.178               80      57m
horizon                     <none>   horizon,horizon.openstack,horizon.openstack.svc.cluster.local                                     10.2.11.177,10.2.11.178               80      99s
keystone                    <none>   keystone,keystone.openstack,keystone.openstack.svc.cluster.local                                  10.2.11.177,10.2.11.178               80      72m
metadata                    <none>   metadata,metadata.openstack,metadata.openstack.svc.cluster.local                                  10.2.11.177,10.2.11.178               80      34m
neutron                     <none>   neutron,neutron.openstack,neutron.openstack.svc.cluster.local                                     10.2.11.177,10.2.11.178               80      9m38s
nova                        <none>   nova,nova.openstack,nova.openstack.svc.cluster.local                                              10.2.11.177,10.2.11.178               80      34m
novncproxy                  <none>   novncproxy,novncproxy.openstack,novncproxy.openstack.svc.cluster.local                            10.2.11.177,10.2.11.178               80      34m
openstack-ingress-openstack <none>   *.openstack.svc.cluster.local                                                                     10.2.11.176,10.2.11.177,10.2.11.178   80      133m
placement                   <none>   placement,placement.openstack,placement.openstack.svc.cluster.local                               10.2.11.177,10.2.11.178               80      34m
rabbitmq-mgr-7b1733         <none>   rabbitmq-mgr-7b1733,rabbitmq-mgr-7b1733.openstack,rabbitmq-mgr-7b1733.openstack.svc.cluster.local 10.2.11.177,10.2.11.178               80      86m

配置/etc/hosts解析

10.2.11.176  k8s-1
10.2.11.177  k8s-2 cinder.openstack.svc.cluster.local glance.openstack.svc.cluster.local horizon.openstack.svc.cluster.local keystone.openstack.svc.cluster.local
10.2.11.178  k8s-3 neutron.openstack.svc.cluster.local nova.openstack.svc.cluster.local placement.openstack.svc.cluster.local

配置admin-rc环境变量

cat admin.rc
export OS_USERNAME=admin
export OS_PASSWORD=password
export OS_PROJECT_NAME=admin
export OS_USER_DOMAIN_NAME=default
export OS_PROJECT_DOMAIN_NAME=default
export OS_AUTH_URL=http://keystone.openstack.svc.cluster.local/v3
export OS_IDENTITY_API_VERSION=3

windows配置解析： C:\Windows\System11\drivers\etc\hosts

10.2.11.178 horizon.openstack.svc.cluster.local
10.2.11.178 novncproxy.openstack.svc.cluster.local
10.2.11.178 traefik-ui.minikube

修改 vnc ingress

kubectl edit ingress novncproxy -n openstack
删除
medatada/annotations下得数据

wen328

关注

1
点赞
踩
7

收藏

觉得还不错? 一键收藏
1
评论
复制链接

分享到 QQ

分享到新浪微博

扫一扫

专栏目录