1 概述
部署pg数据库一主两从,pg版本是v 15.10.0,参考bitnami/postgresql-ha。
2 环境准备
2.1 购买SFS Turbo实例
2.2 购买华为云容器集群CCE实例
2.3 在k8s中创建storageclass对象
参数everest.io/share-access-to是VPC的ID。
参数everest.io/share-export-location是sfs turbo实例的共享路径:自定义子目录,sfs turbo实例的共享路径是在sfs实例的详细页查询,自定义子目录可以是任意路径。
参数everest.io/volume-id是sfs turbo实例的ID。
只需要修改以上三个参数。
在本文,storageclass的名称叫做sfsturbo-subpath-sc。
apiVersion: storage.k8s.io/v1
allowVolumeExpansion: true
kind: StorageClass
metadata:
name: sfsturbo-subpath-sc
mountOptions:
- lock
parameters:
csi.storage.k8s.io/csi-driver-name: sfsturbo.csi.everest.io
csi.storage.k8s.io/fstype: nfs
everest.io/archive-on-delete: "true"
everest.io/share-access-to: xxxxxxxxxxxxxxxxxx # VPC ID
everest.io/share-expand-type: bandwidth
everest.io/share-export-location: xxxxx.sfsturbo.internal:/mydir # sfs turbo实例的共享路径:自定义子目录
everest.io/share-source: sfs-turbo
everest.io/share-volume-type: STANDARD
everest.io/volume-as: subpath
everest.io/volume-id: xxxxxxxxxxxxx # sfs turbo实例的ID
provisioner: everest-csi-provisioner
reclaimPolicy: Retain
volumeBindingMode: Immediate
3 部署
直接apply下面的文件即可,所用的storageclass名称是刚刚创建的sfsturbo-subpath-sc。
账号是postgres,密码是ilovepg@2025。
---
# Source: postgresql-ha/templates/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: pg-postgresql-ha
namespace: "default"
labels:
app.kubernetes.io/instance: pg
automountServiceAccountToken: false
---
# Source: postgresql-ha/templates/pgpool/secrets.yaml
apiVersion: v1
kind: Secret
metadata:
name: pg-postgresql-ha-pgpool
namespace: "default"
labels:
app.kubernetes.io/instance: pg
type: Opaque
data:
admin-password: "VlJrTzV1ajBBSg=="
---
# Source: postgresql-ha/templates/postgresql/secrets.yaml
apiVersion: v1
kind: Secret
metadata:
name: pg-postgresql-ha-postgresql
namespace: "default"
labels:
app.kubernetes.io/instance: pg
type: Opaque
data:
password: "aWxvdmVwZ0AyMDI1"
repmgr-password: "aEticEpxRmJpMQ=="
---
# Source: postgresql-ha/templates/postgresql/hooks-scripts-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: pg-postgresql-ha-postgresql-hooks-scripts
namespace: "default"
labels:
app.kubernetes.io/instance: pg
data:
pre-stop.sh: |-
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
# Debug section
exec 3>&1
exec 4>&2
# Process input parameters
MIN_DELAY_AFTER_PG_STOP_SECONDS=$1
# Load Libraries
. /opt/bitnami/scripts/liblog.sh
. /opt/bitnami/scripts/libpostgresql.sh
. /opt/bitnami/scripts/librepmgr.sh
# Load PostgreSQL & repmgr environment variables
. /opt/bitnami/scripts/postgresql-env.sh
# Auxiliary functions
is_new_primary_ready() {
return_value=1
currenty_primary_node="$(repmgr_get_primary_node)"
currenty_primary_host="$(echo $currenty_primary_node | awk '{print $1}')"
info "$currenty_primary_host != $REPMGR_NODE_NETWORK_NAME"
if [[ $(echo $currenty_primary_node | wc -w) -eq 2 ]] && [[ "$currenty_primary_host" != "$REPMGR_NODE_NETWORK_NAME" ]]; then
info "New primary detected, leaving the cluster..."
return_value=0
else
info "Waiting for a new primary to be available..."
fi
return $return_value
}
export MODULE="pre-stop-hook"
if [[ "${BITNAMI_DEBUG}" == "true" ]]; then
info "Bash debug is on"
else
info "Bash debug is off"
exec 1>/dev/null
exec 2>/dev/null
fi
postgresql_enable_nss_wrapper
# Prepare env vars for managing roles
readarray -t primary_node < <(repmgr_get_upstream_node)
primary_host="${primary_node[0]}"
# Stop postgresql for graceful exit.
PG_STOP_TIME=$EPOCHSECONDS
postgresql_stop
if [[ -z "$primary_host" ]] || [[ "$primary_host" == "$REPMGR_NODE_NETWORK_NAME" ]]; then
info "Primary node need to wait for a new primary node before leaving the cluster"
retry_while is_new_primary_ready 10 5
else
info "Standby node doesn't need to wait for a new primary switchover. Leaving the cluster"
fi
# Make sure pre-stop hook waits at least 25 seconds after stop of PG to make sure PGPOOL detects node is down.
# default terminationGracePeriodSeconds=30 seconds
PG_STOP_DURATION=$(($EPOCHSECONDS - $PG_STOP_TIME))
if (( $PG_STOP_DURATION < $MIN_DELAY_AFTER_PG_STOP_SECONDS )); then
WAIT_TO_PG_POOL_TIME=$(($MIN_DELAY_AFTER_PG_STOP_SECONDS - $PG_STOP_DURATION))
info "PG stopped including primary switchover in $PG_STOP_DURATION. Waiting additional $WAIT_TO_PG_POOL_TIME seconds for PG pool"
sleep $WAIT_TO_PG_POOL_TIME
fi
readiness-probe.sh: |-
#!/bin/bash
set -o errexit
set -o pipefail
set -o nounset
# Debug section
exec 3>&1
exec 4>&2
# Load Libraries
. /opt/bitnami/scripts/liblog.sh
. /opt/bitnami/scripts/libpostgresql.sh
# Load PostgreSQL & repmgr environment variables
. /opt/bitnami/scripts/postgresql-env.sh
# Process input parameters
MIN_DELAY_AFTER_POD_READY_FIRST_TIME=$1
TMP_FIRST_READY_FILE_TS="/tmp/ts-first-ready.mark"
TMP_DELAY_APPLIED_FILE="/tmp/delay-applied.mark"
DB_CHECK_RESULT=$(echo "SELECT 1" | postgresql_execute_print_output "$POSTGRESQL_DATABASE" "$POSTGRESQL_USERNAME" "$POSTGRESQL_PASSWORD" "-h 127.0.0.1 -tA" || echo "command failed")
if [[ "$DB_CHECK_RESULT" == "1" ]]; then
if [[ ! -f "$TMP_DELAY_APPLIED_FILE" ]]; then
# DB up, but initial readiness delay not applied
if [[ -f "$TMP_FIRST_READY_FILE_TS" ]]; then
# calculate delay from the first readiness success
FIRST_READY_TS=$(cat $TMP_FIRST_READY_FILE_TS)
CURRENT_DELAY_SECONDS=$(($EPOCHSECONDS - $FIRST_READY_TS))
if (( $CURRENT_DELAY_SECONDS > $MIN_DELAY_AFTER_POD_READY_FIRST_TIME )); then
# minimal delay of the first readiness state passed - report success and mark delay as applied
touch "$TMP_DELAY_APPLIED_FILE"
else
# minimal delay of the first readiness state not reached yet - report failure
exit 1
fi
else
# first ever readiness test success - store timestamp and report failure
echo $EPOCHSECONDS > $TMP_FIRST_READY_FILE_TS
exit 1
fi
fi
else
# DB test failed - report failure
exit 1
fi
---
# Source: postgresql-ha/templates/pgpool/service.yaml
apiVersion: v1
kind: Service
metadata:
name: pg-postgresql-ha-pgpool
namespace: "default"
labels:
app.kubernetes.io/instance: pg
spec:
type: ClusterIP
sessionAffinity: None
ports:
- name: "postgresql"
port: 5432
targetPort: postgresql
protocol: TCP
nodePort: null
selector:
app.kubernetes.io/instance: pg
app.kubernetes.io/component: pgpool
---
# Source: postgresql-ha/templates/postgresql/service-headless.yaml
apiVersion: v1
kind: Service
metadata:
name: pg-postgresql-ha-postgresql-headless
namespace: "default"
labels:
app.kubernetes.io/instance: pg
spec:
type: ClusterIP
clusterIP: None
publishNotReadyAddresses: false
ports:
- name: "postgresql"
port: 5432
targetPort: postgresql
protocol: TCP
selector:
app.kubernetes.io/instance: pg
app.kubernetes.io/component: postgresql
role: data
---
# Source: postgresql-ha/templates/postgresql/service.yaml
apiVersion: v1
kind: Service
metadata:
name: pg-postgresql-ha-postgresql
namespace: "default"
labels:
app.kubernetes.io/instance: pg
spec:
type: ClusterIP
ports:
- name: "postgresql"
port: 5432
targetPort: postgresql
protocol: TCP
selector:
app.kubernetes.io/instance: pg
role: data
---
# Source: postgresql-ha/templates/pgpool/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: pg-postgresql-ha-pgpool
namespace: "default"
labels:
app.kubernetes.io/instance: pg
app.kubernetes.io/component: pgpool
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/instance: pg
app.kubernetes.io/component: pgpool
template:
metadata:
labels:
app.kubernetes.io/instance: pg
app.kubernetes.io/component: pgpool
spec:
automountServiceAccountToken: false
affinity:
podAffinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/instance: pg
app.kubernetes.io/component: pgpool
topologyKey: kubernetes.io/hostname
weight: 1
nodeAffinity:
securityContext:
fsGroup: 1001
fsGroupChangePolicy: Always
supplementalGroups: []
sysctls: []
serviceAccountName: pg-postgresql-ha
# Auxiliary vars to populate environment variables
containers:
- name: pgpool
image: docker.io/bitnami/pgpool:4.6.0-debian-12-r0
imagePullPolicy: "IfNotPresent"
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsGroup: 1001
runAsNonRoot: true
runAsUser: 1001
seLinuxOptions: {}
seccompProfile:
type: RuntimeDefault
env:
- name: BITNAMI_DEBUG
value: "false"
- name: PGPOOL_BACKEND_NODES
value: 0:pg-postgresql-ha-postgresql-0.pg-postgresql-ha-postgresql-headless:5432,1:pg-postgresql-ha-postgresql-1.pg-postgresql-ha-postgresql-headless:5432,2:pg-postgresql-ha-postgresql-2.pg-postgresql-ha-postgresql-headless:5432,
- name: PGPOOL_SR_CHECK_USER
value: "repmgr"
- name: PGPOOL_SR_CHECK_PASSWORD_FILE
value: "/opt/bitnami/pgpool/secrets/pgpool-sr-check-password"
- name: PGPOOL_SR_CHECK_DATABASE
value: "postgres"
- name: PGPOOL_ENABLE_LDAP
value: "no"
- name: PGPOOL_POSTGRES_USERNAME
value: "postgres"
- name: PGPOOL_POSTGRES_PASSWORD_FILE
value: "/opt/bitnami/pgpool/secrets/pgpool-password"
- name: PGPOOL_ADMIN_USERNAME
value: "admin"
- name: PGPOOL_ADMIN_PASSWORD_FILE
value: "/opt/bitnami/pgpool/secrets/admin-password"
- name: PGPOOL_AUTHENTICATION_METHOD
value: "scram-sha-256"
- name: PGPOOL_ENABLE_LOAD_BALANCING
value: "yes"
- name: PGPOOL_DISABLE_LOAD_BALANCE_ON_WRITE
value: "transaction"
- name: PGPOOL_ENABLE_LOG_CONNECTIONS
value: "no"
- name: PGPOOL_ENABLE_LOG_HOSTNAME
value: "yes"
- name: PGPOOL_ENABLE_LOG_PER_NODE_STATEMENT
value: "no"
- name: PGPOOL_RESERVED_CONNECTIONS
value: '1'
- name: PGPOOL_CHILD_LIFE_TIME
value: ""
- name: PGPOOL_ENABLE_TLS
value: "no"
- name: PGPOOL_HEALTH_CHECK_PSQL_TIMEOUT
value: "6"
envFrom:
ports:
- name: postgresql
containerPort: 5432
protocol: TCP
livenessProbe:
failureThreshold: 5
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
exec:
command:
- /opt/bitnami/scripts/pgpool/healthcheck.sh
readinessProbe:
failureThreshold: 5
initialDelaySeconds: 5
periodSeconds: 5
successThreshold: 1
timeoutSeconds: 5
exec:
command:
- bash
- -ec
- 'PGPASSWORD=$(< $PGPOOL_POSTGRES_PASSWORD_FILE) psql -U "postgres" -d "postgres" -h /opt/bitnami/pgpool/tmp -tA -c "SELECT 1" >/dev/null'
resources:
limits:
cpu: 375m
ephemeral-storage: 2Gi
memory: 384Mi
requests:
cpu: 250m
ephemeral-storage: 50Mi
memory: 256Mi
volumeMounts:
- name: empty-dir
mountPath: /tmp
subPath: tmp-dir
- name: empty-dir
mountPath: /opt/bitnami/pgpool/etc
subPath: app-etc-dir
- name: empty-dir
mountPath: /opt/bitnami/pgpool/conf
subPath: app-conf-dir
- name: empty-dir
mountPath: /opt/bitnami/pgpool/tmp
subPath: app-tmp-dir
- name: empty-dir
mountPath: /opt/bitnami/pgpool/logs
subPath: app-logs-dir
- name: postgresql-password
subPath: pgpool-password
mountPath: /opt/bitnami/pgpool/secrets/pgpool-password
- name: postgresql-password
subPath: pgpool-sr-check-password
mountPath: /opt/bitnami/pgpool/secrets/pgpool-sr-check-password
- name: pgpool-password
subPath: admin-password
mountPath: /opt/bitnami/pgpool/secrets/admin-password
volumes:
- name: empty-dir
emptyDir: {}
- name: postgresql-password
secret:
secretName: pg-postgresql-ha-postgresql
items:
- key: password
path: pgpool-password
- key: repmgr-password
path: pgpool-sr-check-password
- name: pgpool-password
secret:
secretName: pg-postgresql-ha-pgpool
items:
- key: admin-password
path: admin-password
---
# Source: postgresql-ha/templates/postgresql/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: pg-postgresql-ha-postgresql
namespace: "default"
labels:
app.kubernetes.io/instance: pg
app.kubernetes.io/component: postgresql
role: data
spec:
replicas: 3
podManagementPolicy: "Parallel"
serviceName: pg-postgresql-ha-postgresql-headless
updateStrategy:
type: RollingUpdate
selector:
matchLabels:
app.kubernetes.io/instance: pg
app.kubernetes.io/component: postgresql
role: data
template:
metadata:
labels:
app.kubernetes.io/instance: pg
app.kubernetes.io/component: postgresql
role: data
spec:
automountServiceAccountToken: false
affinity:
podAffinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/instance: pg
app.kubernetes.io/name: postgresql-ha
app.kubernetes.io/component: postgresql
topologyKey: kubernetes.io/hostname
weight: 1
nodeAffinity:
securityContext:
fsGroup: 1001
fsGroupChangePolicy: Always
supplementalGroups: []
sysctls: []
serviceAccountName: pg-postgresql-ha
hostNetwork: false
hostIPC: false
containers:
- name: postgresql
image: docker.io/bitnami/postgresql-repmgr:15.10.0
imagePullPolicy: "IfNotPresent"
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
privileged: false
readOnlyRootFilesystem: true
runAsGroup: 1001
runAsNonRoot: true
runAsUser: 1001
seLinuxOptions: {}
seccompProfile:
type: RuntimeDefault
lifecycle:
preStop:
exec:
command:
- /pre-stop.sh
- "25"
# Auxiliary vars to populate environment variables
env:
- name: BITNAMI_DEBUG
value: "false"
# PostgreSQL configuration
- name: POSTGRESQL_VOLUME_DIR
value: "/bitnami/postgresql"
- name: PGDATA
value: "/bitnami/postgresql/data"
- name: POSTGRES_USER
value: "postgres"
- name: POSTGRES_PASSWORD_FILE
value: "/opt/bitnami/postgresql/secrets/password"
- name: POSTGRES_DB
value: "postgres"
- name: POSTGRESQL_LOG_HOSTNAME
value: "true"
- name: POSTGRESQL_LOG_CONNECTIONS
value: "false"
- name: POSTGRESQL_LOG_DISCONNECTIONS
value: "false"
- name: POSTGRESQL_PGAUDIT_LOG_CATALOG
value: "off"
- name: POSTGRESQL_CLIENT_MIN_MESSAGES
value: "error"
- name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES
value: "pgaudit, repmgr"
- name: POSTGRESQL_ENABLE_TLS
value: "no"
- name: POSTGRESQL_PORT_NUMBER
value: "5432"
# Repmgr configuration
- name: REPMGR_PORT_NUMBER
value: "5432"
- name: REPMGR_PRIMARY_PORT
value: "5432"
- name: MY_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: REPMGR_UPGRADE_EXTENSION
value: "no"
- name: REPMGR_PGHBA_TRUST_ALL
value: "no"
- name: REPMGR_MOUNTED_CONF_DIR
value: "/bitnami/repmgr/conf"
- name: REPMGR_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: REPMGR_PARTNER_NODES
value: pg-postgresql-ha-postgresql-0.pg-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local,pg-postgresql-ha-postgresql-1.pg-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local,pg-postgresql-ha-postgresql-2.pg-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local,
- name: REPMGR_PRIMARY_HOST
value: "pg-postgresql-ha-postgresql-0.pg-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local"
- name: REPMGR_NODE_NAME
value: "$(MY_POD_NAME)"
- name: REPMGR_NODE_NETWORK_NAME
value: "$(MY_POD_NAME).pg-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local"
- name: REPMGR_NODE_TYPE
value: "data"
- name: REPMGR_LOG_LEVEL
value: "NOTICE"
- name: REPMGR_CONNECT_TIMEOUT
value: "5"
- name: REPMGR_RECONNECT_ATTEMPTS
value: "2"
- name: REPMGR_RECONNECT_INTERVAL
value: "3"
- name: REPMGR_USERNAME
value: "repmgr"
- name: REPMGR_PASSWORD_FILE
value: "/opt/bitnami/postgresql/secrets/repmgr-password"
- name: REPMGR_DATABASE
value: "repmgr"
- name: REPMGR_FENCE_OLD_PRIMARY
value: "no"
- name: REPMGR_CHILD_NODES_CHECK_INTERVAL
value: "5"
- name: REPMGR_CHILD_NODES_CONNECTED_MIN_COUNT
value: "1"
- name: REPMGR_CHILD_NODES_DISCONNECT_TIMEOUT
value: "30"
envFrom:
ports:
- name: postgresql
containerPort: 5432
protocol: TCP
livenessProbe:
failureThreshold: 6
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
exec:
command:
- bash
- -ec
- 'ps waux | grep "data standby clone" | grep -v grep || PGPASSWORD=$(< $POSTGRES_PASSWORD_FILE) psql -w -U "postgres" -d "postgres" -h 127.0.0.1 -p 5432 -c "SELECT 1"'
readinessProbe:
failureThreshold: 6
initialDelaySeconds: 5
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
exec:
command:
- bash
- -ec
- |
exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432
[ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ]
resources:
limits:
cpu: 375m
ephemeral-storage: 2Gi
memory: 384Mi
requests:
cpu: 250m
ephemeral-storage: 50Mi
memory: 256Mi
volumeMounts:
- name: empty-dir
mountPath: /tmp
subPath: tmp-dir
- name: empty-dir
mountPath: /opt/bitnami/postgresql/conf
subPath: app-conf-dir
- name: empty-dir
mountPath: /opt/bitnami/postgresql/tmp
subPath: app-tmp-dir
- name: empty-dir
mountPath: /opt/bitnami/repmgr/conf
subPath: repmgr-conf-dir
- name: empty-dir
mountPath: /opt/bitnami/repmgr/tmp
subPath: repmgr-tmp-dir
- name: empty-dir
mountPath: /opt/bitnami/repmgr/logs
subPath: repmgr-logs-dir
- name: password
mountPath: /opt/bitnami/postgresql/secrets/
- name: data
mountPath: /bitnami/postgresql
- name: hooks-scripts
mountPath: /pre-stop.sh
subPath: pre-stop.sh
- name: hooks-scripts
mountPath: /readiness-probe.sh
subPath: readiness-probe.sh
volumes:
- name: empty-dir
emptyDir: {}
- name: hooks-scripts
configMap:
name: pg-postgresql-ha-postgresql-hooks-scripts
defaultMode: 0755
- name: password
secret:
secretName: pg-postgresql-ha-postgresql
volumeClaimTemplates:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: data
spec:
accessModes:
- "ReadWriteOnce"
resources:
requests:
storage: "8Gi"
storageClassName: sfsturbo-subpath-sc
kubectl get secret --namespace default pg-postgresql-ha-postgresql -o jsonpath="{.data.password}" | base64 -d && echo
进入容器执行如下命令查看pg复制情况:
psql -h 127.0.0.1 -U postgres --password -c "SELECT * FROM pg_stat_replication;"