mysql operator
Kubernetes operator是一种在Kubernetes环境中管理自定义应用的方法。Go语言是一种非常适合编写Kubernetes operator的语言，因为它具有高效的性能，易于编写和维护的代码，并且Kubernetes的大部分代码都是使用Go语言编写的。
示例
下面是一个简单的示例，演示了如何使用Go语言编写一个Kubernetes operator：
package main
import (
"fmt"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
)
func main() {
// 创建一个Kubernetes客户端
config, err := rest.InClusterConfig()
if err != nil {
panic(err.Error())
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
panic(err.Error())
}
// 每隔一段时间检查pod的状态
for {
pods, err := clientset.CoreV1().Pods("").List(metav1.ListOptions{})
if err != nil {
panic(err.Error())
}
fmt.Printf("There are %d pods in the cluster\n", len(pods.Items))
time.Sleep(10 * time.Second)
}
}在上面的代码中，我们使用了Kubernetes Go客户端库来创建一个Kubernetes客户端，并使用该客户端每隔一段时间检查集群中的pod数量。
这仅仅是一个简单的示例，在实际的Kubernetes operator中，您可能需要执行更复杂的操作，例如监视资源的状态并自动进行修复，创建和删除。
案例1
dbscale operator
针对测试数据库dbscale实现一个operator，基本功能包括：
1.部署集群ClusterDeployment
2.集群状态监控
// 3.集群故障自动转移
第一步先实现集群部署逻辑，整体的控制流程如下：
1.DBScaleCluster CR用于描述用户期望的dbscale集群
2.用户通过kubectl创建DBScaleCluster对象
3.dbscale-operator会watch dbscaleCluster以及其它相关对象，基于集群的状态不断调整DBScale/zookkeeper/mysql等组件的Statefulset、Deployment和Service等对象（即实现一组Kubernetes上的自定义控制器dbscale-controller-manager。这些控制器会不断对比dbscaleCluster对象中记录的期望状态与集群的实际状态，并调增Kubernetes中的资源以驱动集群满足期望状态）
4.Kubernetes的原生控制器根据StatefulSet、Deployment、Job等对象创建更新或删除对应的Pod
具体实现：
1.部署集群
定义dbscaleCluster CR用于描述用户期望的DB集群组件，配置参数，期望状态，拓扑等（具体字段需要与dbscale组沟通确定，此处为示例）
/home/gopath/src/great/greatdb-operator/examples/dbscale.yaml
```yaml
apiVsersion: greatdb/v1alpha1
kind: dbscaleCluster
metadata:
ClusterSpec
# Spec defines the behavior of cluster
dbscale: (dascaleSpec) #
ComponentSpec
ResourceRequirements
serviceAccount
replicas
baseImage
service(ServiceSpec)
maxFailoverCount
storageClassCount
dataSubDir
config
storageVolumes([]StorageVolume)
zookeeper: (zookeeperSpec) #
mysql: (mysqlSpec) #
ClusterStatus
clusterID:
dbscale:
synced
phase
statefulSet
members
peerMembers
leader
failureMembers
image
zookeeper: (zookeeperStatus)
mysql: (mysqlStatus)
```controller
实现集群控制接口
/home/gopath/src/great/greatdb-operator/controllers/dbscale/dbscale_cluster_control.go
package dbscale
import (
"greatdb-operator/apis/wu123.com/v1alpha1"
"greatdb-operator/apis/wu123.com/v1alpha1/defaulting"
"greatdb-operator/controller"
"greatdb-operator/reconciler"
apiequality "k8s.io/apimachinery/pkg/api/equality"
errorutils "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/tools/record"
)
// DBScaleControlInterface implements the control logic for updating DBScaleClusters and their children StatefulSets.
// It is implemented as an interface to allow for extensions that provide different semantics.
// Currently, there is only one implementation.
type ControlInterface interface {
// UpdateCluster implements the control logic for StatefulSet creation, update, and deletion
UpdateDBScaleCluster(*v1alpha1.DBScaleCluster) error
}
// NewDefaultDBScaleClusterControl returns a new instance of the default implementation DBScaleClusterControlInterface that
// implements the documented semantics for DBScaleClusters.
func NewDefaultDBScaleClusterControl(
tcControl controller.DBScaleClusterControlInterface,
zkMemberReconciler reconciler.Reconciler,
mysqlMemberReconciler reconciler.Reconciler,
dbscaleMemberReconciler reconciler.Reconciler,
//reclaimPolicyReconciler reconciler.Reconciler,
metaReconciler reconciler.Reconciler,
//orphanPodsCleaner member.OrphanPodsCleaner,
//pvcCleaner member.PVCCleanerInterface,
//pvcResizer member.PVCResizerInterface,
//pumpMemberManager reconciler.Reconciler,
//tiflashMemberManager reconciler.Reconciler,
//ticdcMemberManager reconciler.Reconciler,
//discoveryManager member.DBScaleDiscoveryManager,
dbscaleClusterStatusReconciler reconciler.Reconciler,
//conditionUpdater DBScaleClusterConditionUpdater,
recorder record.EventRecorder) ControlInterface {
return &defaultDBScaleClusterControl{
tcControl: tcControl,
zookeeperMemberReconciler: zkMemberReconciler,
mysqlMemberReconciler: mysqlMemberReconciler,
dbscaleMemberReconciler: dbscaleMemberReconciler,
//reclaimPolicyReconciler: reclaimPolicyReconciler,
metaReconciler: metaReconciler,
//orphanPodsCleaner: orphanPodsCleaner,
//pvcCleaner: pvcCleaner,
//pvcResizer: pvcResizer,
//pumpMemberManager: pumpMemberManager,
//tiflashMemberManager: tiflashMemberManager,
//ticdcMemberManager: ticdcMemberManager,
//discoveryManager: discoveryManager,
DBScaleClusterStatusReconciler: dbscaleClusterStatusReconciler,
//conditionUpdater: conditionUpdater,
recorder: recorder,
}
}
type defaultDBScaleClusterControl struct {
tcControl controller.DBScaleClusterControlInterface
dbscaleMemberReconciler reconciler.Reconciler
zookeeperMemberReconciler reconciler.Reconciler
mysqlMemberReconciler reconciler.Reconciler
//reclaimPolicyReconciler reconciler.Reconciler
metaReconciler reconciler.Reconciler
//orphanPodsCleaner member.OrphanPodsCleaner
//pvcCleaner member.PVCCleanerInterface
//pvcResizer member.PVCResizerInterface
//discoveryManager member.DBScaleDiscoveryManager
DBScaleClusterStatusReconciler reconciler.Reconciler
//conditionUpdater DBScaleClusterConditionUpdater
recorder record.EventRecorder
}
func (c *defaultDBScaleClusterControl) UpdateDBScaleCluster(dc *v1alpha1.DBScaleCluster) error {
c.defaulting(dc)
var errs []error
oldStatus := dc.Status.DeepCopy()
if err := c.updateDBScaleCluster(dc); err != nil {
errs = append(errs, err)
}
//if err := c.conditionUpdater.Update(dc); err != nil {
// errs = append(errs, err)
//}
if apiequality.Semantic.DeepEqual(&dc.Status, oldStatus) {
return errorutils.NewAggregate(errs)
}
//if _, err := c.tcControl.UpdateDBScaleCluster(dc.DeepCopy(), &dc.Status, oldStatus); err != nil {
// errs = append(errs, err)
//}
return errorutils.NewAggregate(errs)
}
func (c *defaultDBScaleClusterControl) defaulting(dc *v1alpha1.DBScaleCluster) {
defaulting.SetDBScaleClusterDefault(dc)
}
func (c *defaultDBScaleClusterControl) updateDBScaleCluster(dc *v1alpha1.DBScaleCluster) error {
// syncing all PVs managed by operator's reclaim policy to Retain
//if err := c.reclaimPolicyManager.Sync(tc); err != nil {
// return err
//}
//cleaning all orphan pods(pd, MySQL or tiflash which don't have a related PVC) managed by operator
/*
skipReasons, err := c.orphanPodsCleaner.Clean(tc)
if err != nil {
return err
}
if klog.V(10) {
for podName, reason := range skipReasons {
klog.Infof("pod %s of cluster %s/%s is skipped, reason %q", podName, tc.Namespace, tc.Name, reason)
}
}
*/
// reconcile DBScale discovery service
//if err := c.discoveryManager.Reconcile(tc); err != nil {
// return err
//}
// works that should do to making the pd cluster current state match the desired state:
// - create or update the pd service
// - create or update the pd headless service
// - create the pd statefulset
// - sync pd cluster status from pd to DBScaleCluster object
// - set two annotations to the first pd member:
// - label.Bootstrapping
// - label.Replicas
// - upgrade the pd cluster
// - scale out/in the pd cluster
// - failover the pd cluster
if err := c.mysqlMemberReconciler.ReconcileDBScale(dc); err != nil {
return err
}
// works that should do to making the MySQL cluster current state match the desired state:
// - waiting for the pd cluster available(pd cluster is in quorum)
// - create or update MySQL headless service
// - create the MySQL statefulset
// - sync MySQL cluster status from pd to DBScaleCluster object
// - set scheduler labels to MySQL stores
// - upgrade the MySQL cluster
// - scale out/in the MySQL cluster
// - failover the MySQL cluster
if err := c.zookeeperMemberReconciler.ReconcileDBScale(dc); err != nil {
return err
}
// works that should do to making the DBScale cluster current state match the desired state:
// - waiting for the MySQL cluster available(at least one peer works)
// - create or update DBScale headless service
// - create the DBScale statefulset
// - sync DBScale cluster status from pd to DBScaleCluster object
// - upgrade the DBScale cluster
// - scale out/in the DBScale cluster
// - failover the DBScale cluster
if err := c.dbscaleMemberReconciler.ReconcileDBScale(dc); err != nil {
return err
}
// syncing the labels from Pod to PVC and PV, these labels include:
// - label.StoreIDLabelKey
// - label.MemberIDLabelKey
// - label.NamespaceLabelKey
if err := c.metaReconciler.ReconcileDBScale(dc); err != nil {
return err
}
// cleaning the pod scheduling annotation for pd and MySQL
/*
pvcSkipReasons, err := c.pvcCleaner.Clean(tc)
if err != nil {
return err
}
if klog.V(10) {
for pvcName, reason := range pvcSkipReasons {
klog.Infof("pvc %s of cluster %s/%s is skipped, reason %q", pvcName, tc.Namespace, tc.Name, reason)
}
}
// resize PVC if necessary
if err := c.pvcResizer.Resize(tc); err != nil {
return err
}
*/
// syncing the some DBScalecluster status attributes
// - sync DBScaleMonitor reference
return c.DBScaleClusterStatusReconciler.ReconcileDBScale(dc)
}
var _ ControlInterface = &defaultDBScaleClusterControl{}
type ControlInterface interface {
// implements the control logic for StatefulSet creation, update, and deletion
UpdateDBScaleCluster(*v1alpha1.DBScaleCluster) error
}dbscaleClusterControl结构体实现具体的集群控制逻辑
/home/gopath/src/great/greatdb-operator/controllers/dbscale/dbscale_cluster_control.go
func (c *defaultDBScaleClusterControl) UpdateDBScaleCluster(dc *v1alpha1.DBScaleCluster) error {
c.defaulting(dc)
var errs []error
oldStatus := dc.Status.DeepCopy()
if err := c.updateDBScaleCluster(dc); err != nil {
errs = append(errs, err)
}
//if err := c.conditionUpdater.Update(dc); err != nil {
// errs = append(errs, err)
//}
if apiequality.Semantic.DeepEqual(&dc.Status, oldStatus) {
return errorutils.NewAggregate(errs)
}
//if _, err := c.tcControl.UpdateDBScaleCluster(dc.DeepCopy(), &dc.Status, oldStatus); err != nil {
// errs = append(errs, err)
//}
return errorutils.NewAggregate(errs)
}func (c *dbscaleClusterControl) UpdateDBScaleCluster(tc *v1alpha1.DBScaleCluster) error {
// 该方法的主要逻辑是：
// 1.deepcopy集群的old status
// 2.一些pvc/pod 清理工作
// c.discoveryManager.Reconcile(tc)
// 3.协调dbscale组件状态：
// - create or update the dbscale service
// - creaet or update the dbscale headless service
// - create the db statefulset
// - sync dbscale status to DBScaleCluster object
// - upgrade the dbscale cluster
// - failover the dbscal cluster
// 4.协调zookeeper组件状态
// - create or update zookeeper headless service
// - create the zookeeper statefulset
// - sync zookeeper cluster status to DBScaleCluster object
// - upgrade the zookeeper cluster
// - failover the zookeeper cluster
// 5.协调mysql组件状态
// - create or update mysql headless service
// - create the mysql statefulset
// - sync mysql cluster status
// - upgrade the mysql cluster
// - failover the mysql cluster
// syncing the some cluster status attributes
// 6.当前状态与old status比较，若相等则return
// 否则，调用DBScaleClusterControlInterface接口的UpdateDBScaleCluster(tc.DeepCopy(), &tc.Status, oldStatus)方法，
// 以RetryOnConflict方式更新
if _, err := c.tcControl.UpdateDBScaleCluster(tc.DeepCopy(), &tc.Status, oldStatus); err != nil {
errs = append(errs, err)
}
return errutils.NewAggregate(errs)
}controller 实现Run()方法
/home/gopath/src/great/greatdb-operator/controllers/dbscale/dbscale_cluster_controller.go
type Controller struct {
deps *controller.Dependencies
// control returns an interface capable of syncing a cluster.
// Abstracted out for testing.
control ControlInterface
// clusters that need to be synced.
queue workqueue.RateLimitingInterface
}
package dbscale
/*
import (
"fmt"
"time"
"greatdb-operator/apis/wu123.com/v1alpha1"
perrors "github.com/pingcap/errors"
"k8s.io/apimachinery/pkg/api/errors"
apps "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog"
"greatdb-operator/apis/wu123.com/controller"
"greatdb-operator/apis/wu123.com/reconciler/dbscale"
"greatdb-operator/apis/wu123.com/reconciler/meta"
)
// Controller controls DBScaleclusters.
type Controller struct {
deps *controller.Dependencies
// control returns an interface capable of syncing a DBScale cluster.
// Abstracted out for testing.
control ControlInterface
// DBScaleclusters that need to be synced.
queue workqueue.RateLimitingInterface
}
// NewController creates a DBScalecluster controller.
func NewController(deps *controller.Dependencies) *Controller {
c := &Controller{
deps: deps,
control: NewDefaultDBScaleClusterControl(
deps.DBScaleClusterControl,
dbscale.NewDBScaleMemberReconciler(deps),
dbscale.NewZookeeperMemberReconciler(deps),
dbscale.NewMySQLMemberReconciler(deps),
//meta.NewReclaimPolicyReconciler(deps),
meta.NewMetaReconciler(deps),
dbscale.NewDBScaleClusterStatusReconciler(deps),
deps.Recorder,
),
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DBScalecluster"),
}
DBScaleClusterInformer := deps.InformerFactory.Greatopensource().V1alpha1().DBScaleClusters()
statefulsetInformer := deps.KubeInformerFactory.Apps().V1().StatefulSets()
DBScaleClusterInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.enqueueDBScaleCluster,
UpdateFunc: func(old, cur interface{}) {
c.enqueueDBScaleCluster(cur)
},
DeleteFunc: c.enqueueDBScaleCluster,
})
statefulsetInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: c.addStatefulSet,
UpdateFunc: func(old, cur interface{}) {
c.updateStatefulSet(old, cur)
},
DeleteFunc: c.deleteStatefulSet,
})
return c
}
// Run runs the DBScalecluster controller.
func (c *Controller) Run(workers int, stopCh <-chan struct{}) {
defer utilruntime.HandleCrash()
defer c.queue.ShutDown()
klog.Info("Starting DBScalecluster controller")
defer klog.Info("Shutting down DBScalecluster controller")
for i := 0; i < workers; i++ {
go wait.Until(c.worker, time.Second, stopCh)
}
<-stopCh
}
// worker runs a worker goroutine that invokes processNextWorkItem until the the controller's queue is closed
func (c *Controller) worker() {
for c.processNextWorkItem() {
}
}
// processNextWorkItem dequeues items, processes them, and marks them done. It enforces that the syncHandler is never
// invoked concurrently with the same key.
func (c *Controller) processNextWorkItem() bool {
key, quit := c.queue.Get()
if quit {
return false
}
defer c.queue.Done(key)
if err := c.sync(key.(string)); err != nil {
if perrors.Find(err, controller.IsRequeueError) != nil {
klog.Infof("DBScaleCluster: %v, still need sync: %v, requeuing", key.(string), err)
} else {
utilruntime.HandleError(fmt.Errorf("DBScaleCluster: %v, sync failed %v, requeuing", key.(string), err))
}
c.queue.AddRateLimited(key)
} else {
c.queue.Forget(key)
}
return true
}
// sync syncs the given DBScalecluster.
func (c *Controller) sync(key string) error {
startTime := time.Now()
defer func() {
klog.V(4).Infof("Finished syncing DBScaleCluster %q (%v)", key, time.Since(startTime))
}()
ns, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
return err
}
tc, err := c.deps.DBScaleClusterLister.DBScaleClusters(ns).Get(name)
if errors.IsNotFound(err) {
klog.Infof("DBScaleCluster has been deleted %v", key)
return nil
}
if err != nil {
return err
}
return c.syncDBScaleCluster(tc.DeepCopy())
}
func (c *Controller) syncDBScaleCluster(tc *v1alpha1.DBScaleCluster) error {
return c.control.UpdateDBScaleCluster(tc)
}
// enqueueDBScaleCluster enqueues the given DBScalecluster in the work queue.
func (c *Controller) enqueueDBScaleCluster(obj interface{}) {
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
if err != nil {
utilruntime.HandleError(fmt.Errorf("Cound't get key for object %+v: %v", obj, err))
return
}
c.queue.Add(key)
}
// addStatefulSet adds the DBScalecluster for the statefulset to the sync queue
func (c *Controller) addStatefulSet(obj interface{}) {
set := obj.(*apps.StatefulSet)
ns := set.GetNamespace()
setName := set.GetName()
if set.DeletionTimestamp != nil {
// on a restart of the controller manager, it's possible a new statefulset shows up in a state that
// is already pending deletion. Prevent the statefulset from being a creation observation.
c.deleteStatefulSet(set)
return
}
// If it has a ControllerRef, that's all that matters.
tc := c.resolveDBScaleClusterFromSet(ns, set)
if tc == nil {
return
}
klog.V(4).Infof("StatefulSet %s/%s created, DBScaleCluster: %s/%s", ns, setName, ns, tc.Name)
c.enqueueDBScaleCluster(tc)
}
// updateStatefulSet adds the DBScalecluster for the current and old statefulsets to the sync queue.
func (c *Controller) updateStatefulSet(old, cur interface{}) {
curSet := cur.(*apps.StatefulSet)
oldSet := old.(*apps.StatefulSet)
ns := curSet.GetNamespace()
setName := curSet.GetName()
if curSet.ResourceVersion == oldSet.ResourceVersion {
// Periodic resync will send update events for all known statefulsets.
// Two different versions of the same statefulset will always have different RVs.
return
}
// If it has a ControllerRef, that's all that matters.
tc := c.resolveDBScaleClusterFromSet(ns, curSet)
if tc == nil {
return
}
klog.V(4).Infof("StatefulSet %s/%s updated, DBScaleCluster: %s/%s", ns, setName, ns, tc.Name)
c.enqueueDBScaleCluster(tc)
}
// deleteStatefulSet enqueues the DBScalecluster for the statefulset accounting for deletion tombstones.
func (c *Controller) deleteStatefulSet(obj interface{}) {
set, ok := obj.(*apps.StatefulSet)
ns := set.GetNamespace()
setName := set.GetName()
// When a delete is dropped, the relist will notice a statefuset in the store not
// in the list, leading to the insertion of a tombstone object which contains
// the deleted key/value.
if !ok {
tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
utilruntime.HandleError(fmt.Errorf("couldn't get object from tombstone %+v", obj))
return
}
set, ok = tombstone.Obj.(*apps.StatefulSet)
if !ok {
utilruntime.HandleError(fmt.Errorf("tombstone contained object that is not a statefuset %+v", obj))
return
}
}
// If it has a DBScaleCluster, that's all that matters.
tc := c.resolveDBScaleClusterFromSet(ns, set)
if tc == nil {
return
}
klog.V(4).Infof("StatefulSet %s/%s deleted through %v.", ns, setName, utilruntime.GetCaller())
c.enqueueDBScaleCluster(tc)
}
// resolveDBScaleClusterFromSet returns the DBScaleCluster by a StatefulSet,
// or nil if the StatefulSet could not be resolved to a matching DBScaleCluster
// of the correct Kind.
func (c *Controller) resolveDBScaleClusterFromSet(namespace string, set *apps.StatefulSet) *v1alpha1.DBScaleCluster {
controllerRef := metav1.GetControllerOf(set)
if controllerRef == nil {
return nil
}
// We can't look up by UID, so look up by Name and then verify UID.
// Don't even try to look up by Name if it's the wrong Kind.
if controllerRef.Kind != controller.ControllerKind.Kind {
return nil
}
tc, err := c.deps.DBScaleClusterLister.DBScaleClusters(namespace).Get(controllerRef.Name)
if err != nil {
return nil
}
if tc.UID != controllerRef.UID {
// The controller we found with this Name is not the same one that the
// ControllerRef points to.
return nil
}
return tc
}
*/
manager
manager的作用是，分别实现集群各个组件Sync逻辑
需要各个组件实现manager接口
// Manager implements the logic for syncing cluster
type Manager interface {
// Sync implements the logic for syncing cluster.
Sync(*v1alpha1.DBScaleCluster) error
}dbscale_manager需要实现Sync方法，
func (m *dbscaleManager) Sync(dc *v1alpha1.DBScaleCluster) error {
// 1. Sync DBScale Service
// create一个new dbscale Service
// 尝试获取oldSvcTmp, err := ServiceLister.Service(ns).Get(controller.DBScaleMemberName(dcName))
// 如果err是NotFound,则返回 ServiceControl.CreateService(tc, newSvc)
// 否则，判断新旧service是否相等，若不等则调用更新逻辑 ServiceControl.UpdateService(tc, &svc)
// 2.Sync DBScale Headless Service
// 逻辑同Service
// 3.Sync DBScale StatefulSet
// 获取旧的StatefulSet oldDBScaleSetTmp, err := StatefulSetLister.StatefulSets(ns).Get(controller.DBScaleMemberName(dcName))
// 同步集群状态syncDBScaleClusterStatus(dc, oldDBScaleSet)
// 同步ConfigMap cm, err := syncDBScaleConfigMap(dc, oldPDSet)
// create新的StatefulSet newDBScaleSet, err := getNewDBScaleSetForDBScaleCluster(tc, cm)
// 如果上述第一步获取旧StatefulSet NotFound 则StatefulSetControl.CreateStatefulSet(tc, newDBScaleSet)
// 否则updateStatefulSet(StatefulSetControl, dc, newDBScaleSet, oldDBScaleSet)
// 4.failover
// failover.Failover(dc)
}/home/gopath/src/great/greatdb-operator/reconciler/dbscale/dbscale_reconciler.go
package dbscale
import (
"crypto/tls"
"fmt"
"greatdb-operator/apis/wu123.com/controller"
"greatdb-operator/apis/wu123.com/label"
"greatdb-operator/apis/wu123.com/reconciler"
"greatdb-operator/apis/wu123.com/util"
"greatdb-operator/apis/wu123.com/v1alpha1"
"path"
"strings"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
corelisters "k8s.io/client-go/listers/core/v1"
"k8s.io/klog"
"k8s.io/utils/pointer"
)
const (
slowQueryLogVolumeName = "slowlog"
slowQueryLogDir = "/var/log/dbscale"
slowQueryLogFile = slowQueryLogDir + "/slowlog"
// clusterCertPath is where the cert for inter-cluster communication stored (if any)
clusterCertPath = "/var/lib/dbscale-tls"
// serverCertPath is where the dbscale-server cert stored (if any)
serverCertPath = "/var/lib/dbscale-server-tls"
// tlsSecretRootCAKey is the key used in tls secret for the root CA.
// When user use self-signed certificates, the root CA must be provided. We
// following the same convention used in Kubernetes service token.
tlsSecretRootCAKey = corev1.ServiceAccountRootCAKey
)
type dbscaleReconciler struct {
deps *controller.Dependencies
dbscaleStatefulSetIsUpgradingFn func(corelisters.PodLister, *apps.StatefulSet, *v1alpha1.DBScaleCluster) (bool, error)
}
// newDBScaleReconciler returns a *dbscaleReconciler
func NewDBScaleMemberReconciler(deps *controller.Dependencies) reconciler.Reconciler {
return &dbscaleReconciler{
deps: deps,
dbscaleStatefulSetIsUpgradingFn: dbscaleStatefulSetIsUpgrading,
}
}
func (m *dbscaleReconciler) ReconcileGreatDB(_ *v1alpha1.GreatDBCluster) error {
return nil
}
func (r *dbscaleReconciler) ReconcileDBScale(dc *v1alpha1.DBScaleCluster) error {
if dc.Spec.DBScale == nil {
return nil
}
/*
ns := dc.GetNamespace()
dcName := dc.GetName()
if dc.Spec.TiKV != nil && !dc.TiKVIsAvailable() {
return controller.RequeueErrorf("dbscaleCluster: [%s/%s], waiting for TiKV cluster running", ns, dcName)
}
if dc.Spec.Pump != nil {
if !dc.PumpIsAvailable() {
return controller.RequeueErrorf("dbscaleCluster: [%s/%s], waiting for Pump cluster running", ns, dcName)
}
}
*/
// Sync dbscale Headless Service
if err := r.syncDBScaleHeadlessServiceForDBScaleCluster(dc); err != nil {
return err
}
// Sync dbscale Service before syncing dbscale StatefulSet
if err := r.syncDBScaleService(dc); err != nil {
return err
}
if dc.Spec.DBScale.IsTLSClientEnabled() {
if err := r.checkTLSClientCert(dc); err != nil {
return err
}
}
// Sync dbscale StatefulSet
return r.syncDBScaleStatefulSetForDBScaleCluster(dc)
}
func (r *dbscaleReconciler) checkTLSClientCert(dc *v1alpha1.DBScaleCluster) error {
ns := dc.Namespace
secretName := tlsClientSecretName(dc)
secret, err := r.deps.SecretLister.Secrets(ns).Get(secretName)
if err != nil {
return fmt.Errorf("unable to load certificates from secret %s/%s: %v", ns, secretName, err)
}
clientCert, certExists := secret.Data[corev1.TLSCertKey]
clientKey, keyExists := secret.Data[corev1.TLSPrivateKeyKey]
if !certExists || !keyExists {
return fmt.Errorf("cert or key does not exist in secret %s/%s", ns, secretName)
}
_, err = tls.X509KeyPair(clientCert, clientKey)
if err != nil {
return fmt.Errorf("unable to load certificates from secret %s/%s: %v", ns, secretName, err)
}
return nil
}
func (r *dbscaleReconciler) syncDBScaleHeadlessServiceForDBScaleCluster(dc *v1alpha1.DBScaleCluster) error {
if dc.Spec.Paused {
klog.V(4).Infof("dbscale cluster %s/%s is paused, skip syncing for dbscale headless service", dc.GetNamespace(), dc.GetName())
return nil
}
ns := dc.GetNamespace()
dcName := dc.GetName()
newSvc := getNewDBScaleHeadlessServiceForDBScaleCluster(dc)
oldSvcTmp, err := r.deps.ServiceLister.Services(ns).Get(controller.DBScalePeerMemberName(dcName))
if errors.IsNotFound(err) {
err = controller.SetServiceLastAppliedConfigAnnotation(newSvc)
if err != nil {
return err
}
return r.deps.ServiceControl.CreateService(dc, newSvc)
}
if err != nil {
return fmt.Errorf("syncDBScaleHeadlessServiceForDBScaleCluster: failed to get svc %s for cluster %s/%s, error: %s", controller.DBScalePeerMemberName(dcName), ns, dcName, err)
}
oldSvc := oldSvcTmp.DeepCopy()
equal, err := controller.ServiceEqual(newSvc, oldSvc)
if err != nil {
return err
}
if !equal {
svc := *oldSvc
svc.Spec = newSvc.Spec
err = controller.SetServiceLastAppliedConfigAnnotation(&svc)
if err != nil {
return err
}
_, err = r.deps.ServiceControl.UpdateService(dc, &svc)
return err
}
return nil
}
func (r *dbscaleReconciler) syncDBScaleStatefulSetForDBScaleCluster(dc *v1alpha1.DBScaleCluster) error {
ns := dc.GetNamespace()
dcName := dc.GetName()
oldDBScaleSetTemp, err := r.deps.StatefulSetLister.StatefulSets(ns).Get(controller.DBScaleMemberName(dcName))
if err != nil && !errors.IsNotFound(err) {
return fmt.Errorf("syncDBScaleStatefulSetForDBScaleCluster: failed to get sts %s for cluster %s/%s, error: %s", controller.DBScaleMemberName(dcName), ns, dcName, err)
}
setNotExist := errors.IsNotFound(err)
oldDBScaleSet := oldDBScaleSetTemp.DeepCopy()
if err = r.syncDBScaleClusterStatus(dc, oldDBScaleSet); err != nil {
return err
}
if dc.Spec.Paused {
klog.V(4).Infof("dbscale cluster %s/%s is paused, skip syncing for dbscale statefulset", dc.GetNamespace(), dc.GetName())
return nil
}
cm, err := r.syncDBScaleConfigMap(dc, oldDBScaleSet)
if err != nil {
return err
}
newDBScaleSet := getNewDBScaleSetForDBScaleCluster(dc, cm)
if setNotExist {
err = SetStatefulSetLastAppliedConfigAnnotation(newDBScaleSet)
if err != nil {
return err
}
err = r.deps.StatefulSetControl.CreateStatefulSet(dc, newDBScaleSet)
if err != nil {
return err
}
dc.Status.DBScale.StatefulSet = &apps.StatefulSetStatus{}
return nil
}
/*
if r.deps.CLIConfig.AutoFailover {
if r.shouldRecover(dc) {
r.DBScaleFailover.Recover(dc)
} else if dc.DBScaleAllPodsStarted() && !dc.DBScaleAllsReady() {
if err := r.DBScaleFailover.Failover(dc); err != nil {
return err
}
}
}
if !templateEqual(newDBScaleSet, oldDBScaleSet) || dc.Status.DBScale.Phase == v1alpha1.UpgradePhase {
if err := r.DBScaleUpgrader.Upgrade(tc, oldDBScaleSet, newDBScaleSet); err != nil {
return err
}
}
*/
return updateStatefulSet(r.deps.StatefulSetControl, dc, newDBScaleSet, oldDBScaleSet)
}
/*
func (r *dbscaleReconciler) shouldRecover(dc *v1alpha1.DBScaleCluster) bool {
if dc.Status.DBScale.Failures == nil {
return false
}
// If all desired replicas (excluding failover pods) of dbscale cluster are
// healthy, we can perform our failover recovery operation.
// Note that failover pods may fail (e.g. lack of resources) and we don't care
// about them because we're going to delete ther.
for ordinal := range dc.DBScaleStsDesiredOrdinals(true) {
name := fmt.Sprintf("%s-%d", controller.DBScaleName(dc.GetName()), ordinal)
pod, err := r.deps.PodLister.Pods(dc.Namespace).Get(name)
if err != nil {
klog.Errorf("pod %s/%s does not exist: %v", dc.Namespace, name, err)
return false
}
if !podutil.IsPodReady(pod) {
return false
}
status, ok := dc.Status.DBScale.s[pod.Name]
if !ok || !status.Health {
return false
}
}
return true
}
*/
func (r *dbscaleReconciler) syncDBScaleService(dc *v1alpha1.DBScaleCluster) error {
if dc.Spec.Paused {
klog.V(4).Infof("dbscale cluster %s/%s is paused, skip syncing for dbscale service", dc.GetNamespace(), dc.GetName())
return nil
}
newSvc := getNewDBScaleServiceOrNil(dc)
// TODO: delete dbscale service if user remove the service spec deliberately
if newSvc == nil {
return nil
}
ns := newSvc.Namespace
oldSvcTmp, err := r.deps.ServiceLister.Services(ns).Get(newSvc.Name)
if errors.IsNotFound(err) {
err = controller.SetServiceLastAppliedConfigAnnotation(newSvc)
if err != nil {
return err
}
return r.deps.ServiceControl.CreateService(dc, newSvc)
}
if err != nil {
return fmt.Errorf("syncDBScaleService: failed to get svc %s for cluster %s/%s, error: %s", newSvc.Name, ns, dc.GetName(), err)
}
oldSvc := oldSvcTmp.DeepCopy()
util.RetainManagedFields(newSvc, oldSvc)
equal, err := controller.ServiceEqual(newSvc, oldSvc)
if err != nil {
return err
}
annoEqual := util.IsSubMapOf(newSvc.Annotations, oldSvc.Annotations)
isOrphan := metav1.GetControllerOf(oldSvc) == nil
if !equal || !annoEqual || isOrphan {
svc := *oldSvc
svc.Spec = newSvc.Spec
err = controller.SetServiceLastAppliedConfigAnnotation(&svc)
if err != nil {
return err
}
svc.Spec.ClusterIP = oldSvc.Spec.ClusterIP
// apply change of annotations if any
for k, v := range newSvc.Annotations {
svc.Annotations[k] = v
}
// also override labels when adopt orphan
if isOrphan {
svc.OwnerReferences = newSvc.OwnerReferences
svc.Labels = newSvc.Labels
}
_, err = r.deps.ServiceControl.UpdateService(dc, &svc)
return err
}
return nil
}
// syncDBScaleConfigMap syncs the configmap of dbscale
func (r *dbscaleReconciler) syncDBScaleConfigMap(dc *v1alpha1.DBScaleCluster, set *apps.StatefulSet) (*corev1.ConfigMap, error) {
// For backward compatibility, only sync dbscale configmap when .DBScale.config is non-nil
if dc.Spec.DBScale.Config == nil {
return nil, nil
}
newCm, err := getDBScaleConfigMap(dc)
if err != nil {
return nil, err
}
var inUseName string
if set != nil {
inUseName = FindConfigMapVolume(&set.Spec.Template.Spec, func(name string) bool {
return strings.HasPrefix(name, controller.DBScaleMemberName(dc.Name))
})
}
klog.V(3).Info("get dbscale in use config map name: ", inUseName)
err = updateConfigMapIfNeed(r.deps.ConfigMapLister, dc.BaseDBScaleSpec().ConfigUpdateStrategy(), inUseName, newCm)
if err != nil {
return nil, err
}
return r.deps.TypedControl.CreateOrUpdateConfigMap(dc, newCm)
}
func getDBScaleConfigMap(dc *v1alpha1.DBScaleCluster) (*corev1.ConfigMap, error) {
config := dc.Spec.DBScale.Config
if config == nil {
return nil, nil
}
// override CA if tls enabled
if dc.IsTLSClusterEnabled() {
config.Set("security.cluster-ssl-ca", path.Join(clusterCertPath, tlsSecretRootCAKey))
config.Set("security.cluster-ssl-cert", path.Join(clusterCertPath, corev1.TLSCertKey))
config.Set("security.cluster-ssl-key", path.Join(clusterCertPath, corev1.TLSPrivateKeyKey))
}
if dc.Spec.DBScale.IsTLSClientEnabled() {
config.Set("security.ssl-ca", path.Join(serverCertPath, tlsSecretRootCAKey))
config.Set("security.ssl-cert", path.Join(serverCertPath, corev1.TLSCertKey))
config.Set("security.ssl-key", path.Join(serverCertPath, corev1.TLSPrivateKeyKey))
}
confText, err := config.MarshalTOML()
if err != nil {
return nil, err
}
/*
plugins := dc.Spec.DBScale.Plugins
dbscaleStartScriptModel := &DBScaleStartScriptModel{
EnablePlugin: len(plugins) > 0,
PluginDirectory: "/plugins",
PluginList: strings.Join(plugins, ","),
ClusterDomain: dc.Spec.ClusterDomain,
}
if dc.IsHeterogeneous() {
dbscaleStartScriptModel.Path = controller.PDName(dc.Spec.Cluster.Name) + ":2379"
} else {
dbscaleStartScriptModel.Path = "${CLUSTER_NAME}-pd:2379"
}
startScript, err := RenderDBScaleStartScript(dbscaleStartScriptModel)
if err != nil {
return nil, err
}
*/
data := map[string]string{
"config-file": string(confText),
//"startup-script": startScript,
}
name := controller.DBScaleMemberName(dc.Name)
instanceName := dc.GetInstanceName()
dbscaleLabels := label.New().Instance(instanceName).DBScale().Labels()
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: dc.Namespace,
Labels: dbscaleLabels,
OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(dc)},
},
Data: data,
}
return cm, nil
}
func getNewDBScaleServiceOrNil(dc *v1alpha1.DBScaleCluster) *corev1.Service {
svcSpec := dc.Spec.DBScale.Service
if svcSpec == nil {
return nil
}
ns := dc.Namespace
dcName := dc.Name
instanceName := dc.GetInstanceName()
dbscaleSelector := label.New().Instance(instanceName).DBScale()
svcName := controller.DBScaleMemberName(dcName)
dbscaleLabels := dbscaleSelector.Copy().UsedByEndUser().Labels()
portName := "mysql-client"
if svcSpec.PortName != nil {
portName = *svcSpec.PortName
}
ports := []corev1.ServicePort{
{
Name: portName,
Port: 4000,
TargetPort: intstr.FromInt(4000),
Protocol: corev1.ProtocolTCP,
NodePort: svcSpec.GetMySQLNodePort(),
},
}
if svcSpec.ShouldExposeStatus() {
ports = append(ports, corev1.ServicePort{
Name: "status",
Port: 10080,
TargetPort: intstr.FromInt(10080),
Protocol: corev1.ProtocolTCP,
NodePort: svcSpec.GetStatusNodePort(),
})
}
dbscaleSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: svcName,
Namespace: ns,
Labels: dbscaleLabels,
Annotations: copyAnnotations(svcSpec.Annotations),
OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(dc)},
},
Spec: corev1.ServiceSpec{
Type: svcSpec.Type,
Ports: ports,
Selector: dbscaleSelector.Labels(),
},
}
if svcSpec.Type == corev1.ServiceTypeLoadBalancer {
if svcSpec.LoadBalancerIP != nil {
dbscaleSvc.Spec.LoadBalancerIP = *svcSpec.LoadBalancerIP
}
if svcSpec.LoadBalancerSourceRanges != nil {
dbscaleSvc.Spec.LoadBalancerSourceRanges = svcSpec.LoadBalancerSourceRanges
}
}
if svcSpec.ExternalTrafficPolicy != nil {
dbscaleSvc.Spec.ExternalTrafficPolicy = *svcSpec.ExternalTrafficPolicy
}
if svcSpec.ClusterIP != nil {
dbscaleSvc.Spec.ClusterIP = *svcSpec.ClusterIP
}
return dbscaleSvc
}
func getNewDBScaleHeadlessServiceForDBScaleCluster(dc *v1alpha1.DBScaleCluster) *corev1.Service {
ns := dc.Namespace
dcName := dc.Name
instanceName := dc.GetInstanceName()
svcName := controller.DBScalePeerMemberName(dcName)
dbscaleSelector := label.New().Instance(instanceName).DBScale()
dbscaleLabel := dbscaleSelector.Copy().UsedByPeer().Labels()
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: svcName,
Namespace: ns,
Labels: dbscaleLabel,
OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(dc)},
},
Spec: corev1.ServiceSpec{
ClusterIP: "None",
Ports: []corev1.ServicePort{
{
Name: "status",
Port: 10080,
TargetPort: intstr.FromInt(10080),
Protocol: corev1.ProtocolTCP,
},
},
Selector: dbscaleSelector.Labels(),
PublishNotReadyAddresses: true,
},
}
}
func getNewDBScaleSetForDBScaleCluster(dc *v1alpha1.DBScaleCluster, cm *corev1.ConfigMap) *apps.StatefulSet {
ns := dc.GetNamespace()
dcName := dc.GetName()
headlessSvcName := controller.DBScalePeerMemberName(dcName)
basedbscaleSpec := dc.BaseDBScaleSpec()
instanceName := dc.GetInstanceName()
dbscaleConfigMap := controller.MemberConfigMapName(dc, v1alpha1.DBScaleMemberType)
if cm != nil {
dbscaleConfigMap = cm.Name
}
annMount, annVolume := annotationsMountVolume()
volMounts := []corev1.VolumeMount{
annMount,
{Name: "config", ReadOnly: true, MountPath: "/etc/dbscale"},
{Name: "startup-script", ReadOnly: true, MountPath: "/usr/local/bin"},
}
if dc.IsTLSClusterEnabled() {
volMounts = append(volMounts, corev1.VolumeMount{
Name: "dbscale-tls", ReadOnly: true, MountPath: clusterCertPath,
})
}
if dc.Spec.DBScale.IsTLSClientEnabled() {
volMounts = append(volMounts, corev1.VolumeMount{
Name: "dbscale-server-tls", ReadOnly: true, MountPath: serverCertPath,
})
}
vols := []corev1.Volume{
annVolume,
{Name: "config", VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: dbscaleConfigMap,
},
Items: []corev1.KeyToPath{{Key: "config-file", Path: "dbscale.toml"}},
}},
},
{Name: "startup-script", VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: dbscaleConfigMap,
},
Items: []corev1.KeyToPath{{Key: "startup-script", Path: "dbscale_start_script.sh"}},
}},
},
}
if dc.IsTLSClusterEnabled() {
vols = append(vols, corev1.Volume{
Name: "dbscale-tls", VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: util.ClusterTLSSecretName(dcName, label.DBScaleLabelVal),
},
},
})
}
if dc.Spec.DBScale.IsTLSClientEnabled() {
secretName := tlsClientSecretName(dc)
vols = append(vols, corev1.Volume{
Name: "dbscale-server-tls", VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secretName,
},
},
})
}
sysctls := "sysctl -w"
var initContainers []corev1.Container
if basedbscaleSpec.Annotations() != nil {
init, ok := basedbscaleSpec.Annotations()[label.AnnSysctlInit]
if ok && (init == label.AnnSysctlInitVal) {
if basedbscaleSpec.PodSecurityContext() != nil && len(basedbscaleSpec.PodSecurityContext().Sysctls) > 0 {
for _, sysctl := range basedbscaleSpec.PodSecurityContext().Sysctls {
sysctls = sysctls + fmt.Sprintf(" %s=%s", sysctl.Name, sysctl.Value)
}
privileged := true
initContainers = append(initContainers, corev1.Container{
Name: "init",
Image: dc.HelperImage(),
Command: []string{
"sh",
"-c",
sysctls,
},
SecurityContext: &corev1.SecurityContext{
Privileged: &privileged,
},
// Init container resourceRequirements should be equal to app container.
// Scheduling is done based on effective requests/limits,
// which means init containers can reserve resources for
// initialization that are not used during the life of the Pod.
// ref:https://kubernetes.io/docs/concepts/workloads/pods/init-containers/#resources
Resources: controller.ContainerResource(dc.Spec.DBScale.ResourceRequirements),
})
}
}
}
// Init container is only used for the case where allowed-unsafe-sysctls
// cannot be enabled for kubelet, so clean the sysctl in statefulset
// SecurityContext if init container is enabled
podSecurityContext := basedbscaleSpec.PodSecurityContext().DeepCopy()
if len(initContainers) > 0 {
podSecurityContext.Sysctls = []corev1.Sysctl{}
}
var containers []corev1.Container
/*
if dc.Spec.DBScale.ShouldSeparateSlowLog() {
// mount a shared volume and tail the slow log to STDOUT using a sidecar.
vols = append(vols, corev1.Volume{
Name: slowQueryLogVolumeName,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
})
volMounts = append(volMounts, corev1.VolumeMount{Name: slowQueryLogVolumeName, MountPath: slowQueryLogDir})
containers = append(containers, corev1.Container{
Name: v1alpha1.SlowLogTailerType.String(),
Image: dc.HelperImage(),
ImagePullPolicy: dc.HelperImagePullPolicy(),
Resources: controller.ContainerResource(dc.Spec.DBScale.GetSlowLogTailerSpec().ResourceRequirements),
VolumeMounts: []corev1.VolumeMount{
{Name: slowQueryLogVolumeName, MountPath: slowQueryLogDir},
},
Command: []string{
"sh",
"-c",
fmt.Sprintf("touch %s; tail -n0 -F %s;", slowQueryLogFile, slowQueryLogFile),
},
})
}
slowLogFileEnvVal := ""
if dc.Spec.DBScale.ShouldSeparateSlowLog() {
slowLogFileEnvVal = slowQueryLogFile
}
*/
envs := []corev1.EnvVar{
{
Name: "CLUSTER_NAME",
Value: dc.GetName(),
},
{
Name: "TZ",
Value: dc.Spec.Timezone,
},
//{
// Name: "BINLOG_ENABLED",
// Value: strconv.FormatBool(dc.IsdbscaleBinlogEnabled()),
//},
//{
// Name: "SLOW_LOG_FILE",
// Value: slowLogFileEnvVal,
//},
{
Name: "POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.name",
},
},
},
{
Name: "NAMESPACE",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
{
Name: "HEADLESS_SERVICE_NAME",
Value: headlessSvcName,
},
}
c := corev1.Container{
Name: v1alpha1.DBScaleMemberType.String(),
Image: dc.DBScaleImage(),
Command: []string{"/bin/sh", "/usr/local/bin/dbscale_start_script.sh"},
ImagePullPolicy: basedbscaleSpec.ImagePullPolicy(),
Ports: []corev1.ContainerPort{
{
Name: "server",
ContainerPort: int32(4000),
Protocol: corev1.ProtocolTCP,
},
{
Name: "status", // pprof, status, metrics
ContainerPort: int32(10080),
Protocol: corev1.ProtocolTCP,
},
},
VolumeMounts: volMounts,
Resources: controller.ContainerResource(dc.Spec.DBScale.ResourceRequirements),
Env: util.AppendEnv(envs, basedbscaleSpec.Env()),
ReadinessProbe: &corev1.Probe{
Handler: builddbscaleReadinessProbHandler(dc),
InitialDelaySeconds: int32(10),
},
}
if dc.Spec.DBScale.Lifecycle != nil {
c.Lifecycle = dc.Spec.DBScale.Lifecycle
}
containers = append(containers, c)
podSpec := basedbscaleSpec.BuildPodSpec()
podSpec.Containers = append(containers, basedbscaleSpec.AdditionalContainers()...)
podSpec.Volumes = append(vols, basedbscaleSpec.AdditionalVolumes()...)
podSpec.SecurityContext = podSecurityContext
podSpec.InitContainers = initContainers
podSpec.ServiceAccountName = dc.Spec.DBScale.ServiceAccount
if podSpec.ServiceAccountName == "" {
podSpec.ServiceAccountName = dc.Spec.ServiceAccount
}
if basedbscaleSpec.HostNetwork() {
podSpec.DNSPolicy = corev1.DNSClusterFirstWithHostNet
}
dbscaleLabel := label.New().Instance(instanceName).DBScale()
podAnnotations := CombineAnnotations(controller.AnnProm(10080), basedbscaleSpec.Annotations())
stsAnnotations := getStsAnnotations(dc.Annotations, label.DBScaleLabelVal)
updateStrategy := apps.StatefulSetUpdateStrategy{}
if basedbscaleSpec.StatefulSetUpdateStrategy() == apps.OnDeleteStatefulSetStrategyType {
updateStrategy.Type = apps.OnDeleteStatefulSetStrategyType
} else {
updateStrategy.Type = apps.RollingUpdateStatefulSetStrategyType
updateStrategy.RollingUpdate = &apps.RollingUpdateStatefulSetStrategy{
Partition: pointer.Int32Ptr(dc.DBScaleStsDesiredReplicas()),
}
}
dbscaleSet := &apps.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: controller.DBScaleMemberName(dcName),
Namespace: ns,
Labels: dbscaleLabel.Labels(),
Annotations: stsAnnotations,
OwnerReferences: []metav1.OwnerReference{controller.GetOwnerRef(dc)},
},
Spec: apps.StatefulSetSpec{
Replicas: pointer.Int32Ptr(dc.DBScaleStsDesiredReplicas()),
Selector: dbscaleLabel.LabelSelector(),
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: dbscaleLabel.Labels(),
Annotations: podAnnotations,
},
Spec: podSpec,
},
ServiceName: controller.DBScalePeerMemberName(dcName),
PodManagementPolicy: apps.ParallelPodManagement,
UpdateStrategy: updateStrategy,
},
}
return dbscaleSet
}
func (r *dbscaleReconciler) syncDBScaleClusterStatus(dc *v1alpha1.DBScaleCluster, set *apps.StatefulSet) error {
if set == nil {
// skip if not created yet
return nil
}
dc.Status.DBScale.StatefulSet = &set.Status
upgrading, err := r.dbscaleStatefulSetIsUpgradingFn(r.deps.PodLister, set, dc)
if err != nil {
return err
}
if dc.DBScaleStsDesiredReplicas() != *set.Spec.Replicas {
dc.Status.DBScale.Phase = v1alpha1.ScalePhase
} else if upgrading && dc.Status.MySQL.Phase != v1alpha1.UpgradePhase &&
dc.Status.Zookeeper.Phase != v1alpha1.UpgradePhase {
dc.Status.DBScale.Phase = v1alpha1.UpgradePhase
} else {
dc.Status.DBScale.Phase = v1alpha1.NormalPhase
}
dbscaleStatus := map[string]v1alpha1.DBScaleMember{}
for id := range v1alpha1.GetPodOrdinals(dc.Status.DBScale.StatefulSet.Replicas, set) {
name := fmt.Sprintf("%s-%d", controller.DBScaleMemberName(dc.GetName()), id)
health, err := r.deps.DBScaleControl.GetHealth(dc, int32(id))
if err != nil {
return err
}
newDBScale := v1alpha1.DBScaleMember{
Name: name,
Health: health,
}
oldDBScale, exist := dc.Status.DBScale.Members[name]
newDBScale.LastTransitionTime = metav1.Now()
if exist {
newDBScale.NodeName = oldDBScale.NodeName
if oldDBScale.Health == newDBScale.Health {
newDBScale.LastTransitionTime = oldDBScale.LastTransitionTime
}
}
pod, err := r.deps.PodLister.Pods(dc.GetNamespace()).Get(name)
if err != nil && !errors.IsNotFound(err) {
return fmt.Errorf("syncDBScaleClusterStatus: failed to get pods %s for cluster %s/%s, error: %s", name, dc.GetNamespace(), dc.GetName(), err)
}
if pod != nil && pod.Spec.NodeName != "" {
// Update assiged node if pod exists and is scheduled
newDBScale.NodeName = pod.Spec.NodeName
}
dbscaleStatus[name] = newDBScale
}
dc.Status.DBScale.Members = dbscaleStatus
dc.Status.DBScale.Image = ""
c := filterContainer(set, "dbscale")
if c != nil {
dc.Status.DBScale.Image = c.Image
}
return nil
}
func dbscaleStatefulSetIsUpgrading(podLister corelisters.PodLister, set *apps.StatefulSet, dc *v1alpha1.DBScaleCluster) (bool, error) {
if statefulSetIsUpgrading(set) {
return true, nil
}
selector, err := label.New().
Instance(dc.GetInstanceName()).
DBScale().
Selector()
if err != nil {
return false, err
}
dbscalePods, err := podLister.Pods(dc.GetNamespace()).List(selector)
if err != nil {
return false, fmt.Errorf("dbscaleStatefulSetIsUpgrading: failed to get pods for cluster %s/%s, selector %s, error: %s", dc.GetNamespace(), dc.GetInstanceName(), selector, err)
}
for _, pod := range dbscalePods {
revisionHash, exist := pod.Labels[apps.ControllerRevisionHashLabelKey]
if !exist {
return false, nil
}
if revisionHash != dc.Status.DBScale.StatefulSet.UpdateRevision {
return true, nil
}
}
return false, nil
}
func builddbscaleReadinessProbHandler(dc *v1alpha1.DBScaleCluster) corev1.Handler {
if dc.Spec.DBScale.ReadinessProbe != nil {
if tp := dc.Spec.DBScale.ReadinessProbe.Type; tp != nil {
if *tp == v1alpha1.CommandProbeType {
command := builddbscaleProbeCommand(dc)
return corev1.Handler{
Exec: &corev1.ExecAction{
Command: command,
},
}
}
}
}
// fall to default case v1alpha1.TCPProbeType
return corev1.Handler{
TCPSocket: &corev1.TCPSocketAction{
Port: intstr.FromInt(4000),
},
}
}
func builddbscaleProbeCommand(dc *v1alpha1.DBScaleCluster) (command []string) {
host := "127.0.0.1"
readinessURL := fmt.Sprintf("%s://%s:10080/status", dc.Scheme(), host)
command = append(command, "curl")
command = append(command, readinessURL)
// Fail silently (no output at all) on server errors
// without this if the server return 500, the exist code will be 0
// and probe is success.
command = append(command, "--fail")
// follow 301 or 302 redirect
command = append(command, "--location")
if dc.IsTLSClusterEnabled() {
cacert := path.Join(clusterCertPath, tlsSecretRootCAKey)
cert := path.Join(clusterCertPath, corev1.TLSCertKey)
key := path.Join(clusterCertPath, corev1.TLSPrivateKeyKey)
command = append(command, "--cacert", cacert)
command = append(command, "--cert", cert)
command = append(command, "--key", key)
}
return
}
func tlsClientSecretName(dc *v1alpha1.DBScaleCluster) string {
return fmt.Sprintf("%s-server-secret", controller.DBScaleMemberName(dc.Name))
}
zookeeper_manager
逻辑同上
/home/gopath/src/great/greatdb-operator/reconciler/dbscale/zookeeper_reconciler.go
mysql_manager
逻辑同上
/home/gopath/src/great/greatdb-operator/reconciler/dbscale/mysql_reconciler.go
辅助controller
方便自定义控制器通过k8s原生控制器实现pod的创建/删除等操作
// ServiceControlInterface manages Services used in Cluster
type ServiceControlInterface interface {
CreateService(runtime.Object, *corev1.Service) error
UpdateService(runtime.Object, *corev1.Service)(*corev1.Service, error)
DeleteService(runtime.Object, *corev1.Service) error
}// StatefulSetControlInterface defines the interface that users to create, update, and delete StatefulSets,
type StatefulSetControlInterface interface {
CreateStatefulSet(runtime.Object, *apps.StatefulSet) error
UpdateStatefulSet(runtime.Object, *apps.StatefulSet) (*apps.StatefulSet, errors)
DeleteStatefulSet(runtime.Object, *apps.StatefulSet) error
}/home/gopath/src/great/greatdb-operator/controllers/service_control.go
/home/gopath/src/great/greatdb-operator/controllers/statefulset_control.go
2.集群状态监控
3.故障自动转移
各个组件（dbscale,zookeeper,mysql）的manager需要提供Failover接口的功能
// Failover implements the logic for dbscale/zookeeper/mysql's failover and recovery.
type Failover interface {
Failover(*v1alpha1.DBScaleCluster) error
Recover(*v1alpha1.DBScaleCluster)
RemoveUndesiredFailures(*v1alpha1.DBScaleCluster)
}
Failover方法的主要逻辑是轮询各个组件的状态，并标记处于failure状态的成员。
greatdbcluster5.0集群上k8s方案
方案：两个
1. 参考dbscale当前的实现方式：编写yaml配置文件使用statefulset部署5.0组件，使用job对象初始化集群。
需要准备或实现的内容包括：
`5.0的镜像`：可以使用一个centos/debian等操作系统镜像为基础将5.0拷进去
`configmap.yaml文件`:使用一个k8s configmap对象管理5.0配置文件，基本内容包括
```yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
name:
labels:
data:
datanode.cnf: |
[mysqld]
server_id = %d
loose-group_replication_local_address = %s:1%d
datadir=/var/lib/greatdb-cluster/
default_authentication_plugin=mysql_native_password
general_log=1
general_log_file=/var/lib/greatdb-cluster/general.log
log_error=/var/lib/greatdb-cluster/error.log
max_connections=1000
port=%d
socket=/var/lib/greatdb-cluster/mysql.sock
sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES
user=greatdb
# group replication configuration
binlog-checksum=NONE
enforce-gtid-consistency
gtid-mode=ON
loose-group_replication_start_on_boot=OFF
loose_group_replication_recovery_get_public_key=ON
sqlnode.cnf: |
[mysqld]
server_id = %d
loose-group_replication_local_address = %s:1%d
datadir=/var/lib/greatdb-cluster/
default_authentication_plugin=mysql_native_password
general_log=1
general_log_file=/var/lib/greatdb-cluster/general.log
log_error=/var/lib/greatdb-cluster/error.log
max_connections=1000
#mysqlx_port = 330600
port=%d
socket=/var/lib/greatdb-cluster/greatdb.sock
sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES
user=greatdb
# group replication configuration
binlog-checksum=NONE
enforce-gtid-consistency
gtid-mode=ON
loose-group_replication_start_on_boot=OFF
loose_group_replication_recovery_get_public_key=ON
loose_group_replication_recovery_retry_count=100
````statefulset.yaml文件`: 主要用于配置一个shard 或 一个sqlnode集群，几个主要的点包括：
```yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name:
labels:
spec:
selector:
serviceName:
replicas: # ----------------------------------------------1. 配置sqlnode节点个数，或每个shard的datanode节点个数
selector:
template:
metadata:
spec:
initContainers: # ------------------------------------2. pod初始化容器用于初始化配置文件datanode.cnf/sqlnode.cnf
- name: init-greatdb
image:
imagePullPolicy:
command:
- bash
- "-c"
- |
set -ex
#
env:
- name: MYSQL_REPLICATION_USER
value:
- name: MYSQL_REPLICATION_PASSWORD
valueFrom:
secretKeyRef:
name:
key: mysql-replication-password
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
containers: # ------------------------------------3. pod主容器用于初始启动SQLNode/Datanode,若非首次启动则只启动跳过初始化
- name: mysql
image:
imagePullPolicy:
env:
- name: MYSQL_DATABASE
value:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: {{ template "fullname" . }}
key: mysql-root-password
- name: MYSQL_REPLICATION_USER
value: {{ .Values.mysqlha.mysqlReplicationUser }}
- name: MYSQL_REPLICATION_PASSWORD
valueFrom:
secretKeyRef:
name: {{ template "fullname" . }}
key: mysql-replication-password
{{ if .Values.mysqlha.mysqlUser }}
- name: MYSQL_USER
value: {{ .Values.mysqlha.mysqlUser | quote }}
- name: MYSQL_PASSWORD
valueFrom:
secretKeyRef:
name: {{ template "fullname" . }}
key: mysql-password
{{ end }}
ports:
- name: mysql
containerPort: 3306
volumeMounts:
- name: data
mountPath: /var/lib/mysql
subPath: mysql
- name: conf
mountPath: /etc/mysql/conf.d
resources:
requests:
cpu: {{ .Values.resources.requests.cpu }}
memory: {{ .Values.resources.requests.memory }}
livenessProbe:
exec:
command:
- /bin/sh
- "-c"
- mysqladmin ping -h 127.0.0.1 -u root -p${MYSQL_ROOT_PASSWORD}
initialDelaySeconds: 30
timeoutSeconds: 5
readinessProbe:
exec:
# Check we can execute queries over TCP (skip-networking is off).
command:
- /bin/sh
- "-c"
- MYSQL_PWD="${MYSQL_ROOT_PASSWORD}"
- mysql -h 127.0.0.1 -u root -e "SELECT 1"
initialDelaySeconds: 10
timeoutSeconds: 1
- name: metrics # ------------------------------------------------4. pod监控
image:
imagePullPolicy: {{ .Values.imagePullPolicy | quote }}
{{- if .Values.mysqlha.mysqlAllowEmptyPassword }}
command: ['sh', '-c', 'DATA_SOURCE_NAME="root@(localhost:3306)/" /bin/mysqld_exporter' ]
{{- else }}
env:
- name: MYSQL_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: {{ template "fullname" . }}
key: mysql-root-password
command: [ 'sh', '-c', 'DATA_SOURCE_NAME="root:$MYSQL_ROOT_PASSWORD@(localhost:3306)/" /bin/mysqld_exporter' ]
{{- end }}
ports:
- name: metrics
containerPort: 9104
livenessProbe:
httpGet:
path: /
port: metrics
initialDelaySeconds: {{ .Values.metrics.livenessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.metrics.livenessProbe.timeoutSeconds }}
readinessProbe:
httpGet:
path: /
port: metrics
initialDelaySeconds: {{ .Values.metrics.readinessProbe.initialDelaySeconds }}
timeoutSeconds: {{ .Values.metrics.readinessProbe.timeoutSeconds }}
resources:
{{ toYaml .Values.metrics.resources | indent 10 }}
{{- end }}
volumes:
- name: conf
emptyDir: {}
- name: config-map #--------------------------------------------- 5. 配置文件configmap
configMap:
name: {{ template "fullname" . }}
- name: scripts
emptyDir: {}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates: #------------------------------------------------ 6. 配置持久卷，可以使用https://github.com/rancher/local-path-provisioner
# 或https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner （在部署集群之前安装准备好即可）
- metadata:
name: data
annotations:
{{- range $key, $value := .Values.persistence.annotations }}
{{ $key }}: {{ $value }}
{{- end }}
spec:
accessModes:
{{- range .Values.persistence.accessModes }}
- {{ . | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.persistence.size | quote }}
{{- if .Values.persistence.storageClass }}
{{- if (eq "-" .Values.persistence.storageClass) }}
storageClassName: ""
{{- else }}
storageClassName: "{{ .Values.persistence.storageClass }}"
{{- end }}
{{- end }}
{{- else }}
- name: data
emptyDir: {}
{{- end }}
````service.yaml文件`: 用于配置service对象，实现外部对shard或sqlnode的访问，形如
```yaml
apiVersion: v1
kind: Service
metadata:
name: {{ template "fullname" . }}
labels:
spec:
ports:
- name: {{ template "fullname" . }}
port: 3306
clusterIP: None
selector:
app: {{ template "fullname" . }}
````job.yaml文件`：创建一个job对象用以执行初始化集群的工作,使用一个mysql-client或者greatdb镜像，连接sqlnode，执行greatdb_init_cluster/greatdb_add_sqlnode/greatdb_add_datanode/greatdb_init_shard等操作
`其他`:监控可以使用mysql-exporter; 存储可以使用现有的PV解决方案比如：[https://github.com/rancher/local-path-provisioner](https://github.com/rancher/local-path-provisioner) 或 [https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner](https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner)
将上述内容整理为helm chart之后部署流程为:
```shell
helm install shard1 datanode
helm install shard2 datanode
...
helm install shardn datanode
helm install sqlnode sqlnode
```2. greatdb-operator方式
最基本的实现集群部署逻辑
案例2
greatdb-operator设计实现
1.基本设计
greatdb-operator的实现参考了tidb-operator, mysql-operator等实现，就是标准的“CRD+控制逻辑controller”的k8s operator模式，用于协调自定义的GreatDBCluster/GreatDBMonitor等资源
具体的实现内容/接口定义可见代码仓库
github.com/yunixiangfeng/great/tree/main/greatdb-operator
kubernetes v1.24.2
go v1.18.5
案例 GreatDBCluster CRD
1.使用kubebuilder生成greatdbcluster crd
cd /home/gopath/src/
mkdir greatdb-operator
cd greatdb-operator
// go mod init greatdb-operator
// kubebuilder init --domain wu123.com
kubebuilder init --plugins go/v3 --domain wu123.com --owner "wu123"
kubebuilder edit --multigroup=true
// kubebuilder create api --group wu123.com --version v1alpha1 --kind GreatDBCluster
kubebuilder create api --group greatdb --version v1alpha1 --kind GreatDBCluster
make install
// /home/gopath/src/greatdb-operator/bin/controller-gen "crd:trivialVersions=true" rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases[root@k8s-worker02 greatdb-operator]# go mod init greatdb-operator
go: creating new go.mod: module greatdb-operator
[root@k8s-worker02 greatdb-operator]# kubebuilder init --plugins go/v3 --domain wu123.com --owner "wu123"
Writing kustomize manifests for you to edit...
Writing scaffold for you to edit...
Get controller runtime:
$ go get sigs.k8s.io/controller-runtime@v0.12.2
Update dependencies:
$ go mod tidy
Next: define a resource with:
$ kubebuilder create api项目目录
[root@k8s-worker02 greatdb-operator]# kubebuilder create api --group greatdb --version v1alpha1 --kind GreatDBCluster
Create Resource [y/n]
y
Create Controller [y/n]
y
Writing kustomize manifests for you to edit...
Writing scaffold for you to edit...
apis/wu123.com/v1alpha1/greatdbcluster_types.go
controllers/wu123.com/greatdbcluster_controller.go
Update dependencies:
$ go mod tidy
Running make:
$ make generate
mkdir -p /home/gopath/src/greatdb-operator/bin
test -s /home/gopath/src/greatdb-operator/bin/controller-gen || GOBIN=/home/gopath/src/greatdb-operator/bin go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2
go: sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2: sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2: Get "https://goproxy.io/sigs.k8s.io/controller-tools/cmd/controller-gen/@v/v0.9.2.info": unexpected EOF
make: *** [Makefile:127: /home/gopath/src/greatdb-operator/bin/controller-gen] Error 1
Error: failed to create API: unable to run post-scaffold tasks of "base.go.kubebuilder.io/v3": exit status 2
Usage:
kubebuilder create api [flags]
Examples:
# Create a frigates API with Group: ship, Version: v1beta1 and Kind: Frigate
kubebuilder create api --group ship --version v1beta1 --kind Frigate
# Edit the API Scheme
nano api/v1beta1/frigate_types.go
# Edit the Controller
nano controllers/frigate/frigate_controller.go
# Edit the Controller Test
nano controllers/frigate/frigate_controller_test.go
# Generate the manifests
make manifests
# Install CRDs into the Kubernetes cluster using kubectl apply
make install
# Regenerate code and run against the Kubernetes cluster configured by ~/.kube/config
make run
Flags:
--controller if set, generate the controller without prompting the user (default true)
--force attempt to create resource even if it already exists
--group string resource Group
-h, --help help for api
--kind string resource Kind
--make make generate if true, run make generate after generating files (default true)
--namespaced resource is namespaced (default true)
--plural string resource irregular plural form
--resource if set, generate the resource without prompting the user (default true)
--version string resource Version
Global Flags:
--plugins strings plugin keys to be used for this subcommand execution
2023/06/28 20:55:50 failed to create API: unable to run post-scaffold tasks of "base.go.kubebuilder.io/v3": exit status 2[root@k8s-worker02 bin]# find / -name controller-gen
find: ‘/run/user/1000/gvfs’: Permission denied
find: ‘/tmp/.mount_Qv2rayn36GfD’: Permission denied
/home/gopath/pkg/mod/sigs.k8s.io/controller-tools@v0.9.2/cmd/controller-gen
/home/gopath/src/k8s-exercise/k8s-operator/bin/controller-gen
/home/gopath/src/k8s-exercise/elasticweb/bin/controller-gen
/home/gopath/src/helloworld/bin/controller-gen
/home/gopath/elasticweb/bin/controller-gen
[root@k8s-worker02 bin]# cp /home/gopath/elasticweb/bin/controller-gen /home/gopath/src/greatdb-operator/bin/
cp: failed to access ‘/home/gopath/src/greatdb-operator/bin/’: Not a directory
[root@k8s-worker02 bin]# cp /home/gopath/elasticweb/bin/controller-gen /home/gopath/src/greatdb-operator/bin/
[root@k8s-worker02 greatdb-operator]# make install
/home/gopath/src/greatdb-operator/bin/controller-gen rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
test -s /home/gopath/src/greatdb-operator/bin/kustomize || { curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s -- 3.8.7 /home/gopath/src/greatdb-operator/bin; }
make: *** [Makefile:122: /home/gopath/src/greatdb-operator/bin/kustomize] Error 35
[root@k8s-worker02 bin]# cp /home/gopath/elasticweb/bin/kustomize /home/gopath/src/greatdb-operator/bin/
[root@k8s-worker02 greatdb-operator]# make install
test -s /home/gopath/src/greatdb-operator/bin/controller-gen || GOBIN=/home/gopath/src/greatdb-operator/bin go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2
/home/gopath/src/greatdb-operator/bin/controller-gen rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
/home/gopath/src/greatdb-operator/bin/kustomize build config/crd | kubectl apply -f -
customresourcedefinition.apiextensions.k8s.io/greatdbclusters.wu123.com.wu123.com created项目目录
2.使用kubebuilder生成greatdbmonitor crd自定义资源定义
cd /home/gopath/src/greatdb-operator
kubebuilder create api --group greatdb --version v1alpha1 --kind GreatDBMonitor
// /home/gopath/bin/controller-gen "crd:trivialVersions=true" rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
make install[root@k8s-worker02 greatdb-operator]# kubebuilder create api --group greatdb --version v1alpha1 --kind GreatDBMonitor
Create Resource [y/n]
y
Create Controller [y/n]
y
Writing kustomize manifests for you to edit...
Writing scaffold for you to edit...
apis/wu123.com/v1alpha1/greatdbmonitor_types.go
controllers/wu123.com/greatdbmonitor_controller.go
Update dependencies:
$ go mod tidy
Running make:
$ make generate
test -s /home/gopath/src/greatdb-operator/bin/controller-gen || GOBIN=/home/gopath/src/greatdb-operator/bin go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2
/home/gopath/src/greatdb-operator/bin/controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..."
Next: implement your new API and generate the manifests (e.g. CRDs,CRs) with:
$ make manifests[root@k8s-worker02 greatdb-operator]# make install
test -s /home/gopath/src/greatdb-operator/bin/controller-gen || GOBIN=/home/gopath/src/greatdb-operator/bin go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.9.2
/home/gopath/src/greatdb-operator/bin/controller-gen rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
/home/gopath/src/greatdb-operator/bin/kustomize build config/crd | kubectl apply -f -
customresourcedefinition.apiextensions.k8s.io/greatdbclusters.wu123.com.wu123.com unchanged
customresourcedefinition.apiextensions.k8s.io/greatdbmonitors.wu123.com.wu123.com created3.安装crd
创建 /home/gopath/src/greatdb-operator/manifests/greatdbcluster_crd.yaml
/home/gopath/src/greatdb-operator/manifests/greatdbmonitor_crd.yaml
cd /home/gopath/src/greatdb-operator/manifests/ && kubectl create -f greatdbcluster_crd.yaml
cd /home/gopath/src/greatdb-operator/manifests/ && kubectl create -f greatdbmonitor_crd.yaml
[root@k8s-worker02 greatdb-operator]# kubectl create -f manifests/greatdbcluster_crd.yaml
resource mapping not found for name: "greatdbmonitors." namespace: "" from "manifests/greatdbcluster_crd.yaml": no matches for kind "CustomResourceDefinition" in version "apiextensions.k8s.io/v1beta1"
ensure CRDs are installed first
resource mapping not found for name: "greatdbmonitors.wu123.com" namespace: "" from "manifests/greatdbcluster_crd.yaml": no matches for kind "CustomResourceDefinition" in version "apiextensions.k8s.io/v1beta1"
ensure CRDs are installed first修改no matches for kind "CustomResourceDefinition" in version "apiextensions.k8s.io/v1beta1"
为apiextensions.k8s.io/v1
[root@k8s-worker02 greatdb-operator]# kubectl create -f manifests/greatdbcluster_crd.yaml
error: error validating "manifests/greatdbcluster_crd.yaml": error validating data: ValidationError(CustomResourceDefinition): missing required field "spec" in io.k8s.apiextensions-apiserver.pkg.apis.apiextensions.v1.CustomResourceDefinition; if you choose to ignore these errors, turn validation off with --validate=false[root@k8s-worker02 greatdb-operator]# kubectl create -f manifests/greatdbcluster_crd.yaml
customresourcedefinition.apiextensions.k8s.io/greatdbclusters.greatopensource.com created
[root@k8s-worker02 greatdb-operator]# kubectl apply -f manifests/greatdbmonitor_crd.yaml
customresourcedefinition.apiextensions.k8s.io/greatdbmonitors.wu123.com created具体的实现内容/接口定义
/home/gopath/src/great/greatdb-operator/apis/wu123.com/v1alpha1/greatdbcluster_types.go
首先自定义资源类型GreatDBCluster
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
// +k8s:openapi-gen=true
// GreatDBCluster is the control script's spec
type GreatDbCluster struct {
metav1.TypeMeta `json:",inline"`
// +k8s:openapi-gen=false
metav1.ObjectMeta `json:"metadata"`
// Spec defines the behavior of a GreatDB cluster
Spec GreatDbClusterSpec `json:"spec"`
// +k8s:openapi-gen=false
// Most recently observed status of the GreatDB cluster
Status GreatDBClusterStatus `json:"status"`
}
// +k8s:deepcopy-gen:interfaces=k8s.io/apimechinery/pkg/runtime.Object
// +k8s:openapi-gen=true
// GreatDBClusterList is GreatDBCluster list
type GreatDBClusterList struct {
metav1.TypeMeta `json:",inline"`
// +k8s:openapi-gen=false
metav1.ListMeta `json:"items"`
}其中Spec（GreatDBClusterSpec）用以描述集群sqlnode和shard的期望状态，以及一些公共的配置
// +k8s:openapi-gen=true
// GreatDBClusterSpec describe the attributes that a user creates on a GreatDB cluster
type GreatDBClusterSpec struct {
// SQLNode cluster spec
// +optional
// SQLNode *SQLNodeSpec `json:"sqlnode,omitempty"`
// DataNode cluster spec
// +optional
// DataNode *DataNodeSpec `json:"datanode,omitempty"`
// SchedulerName of GreatDB cluster Pods
// +kubebuilder:default=GreatDb-scheduler
// SchedulerName string `json:"schedulerName,omitempty"`
// ...
// Discovery spec
//Discovery DiscoverySpec `json:"discovery,omitempty"`
// Specify a Service Account
ServiceAccount string `json:"serviceAccount,omitempty"`
// SQLNode cluster spec
// +optional
SQLNode *SQLNodeSpec `json:"sqlNode,omitempty"`
// GreatDB cluster spec
// +optional
DataNode *DataNodeSpec `json:"dataNode,omitempty"`
// DataNode cluster spec
// +optional
Shard *ShardSpec `json:"shard,omitempty"`
// Indicates that the GreatDB cluster is paused and will not be processed by
// the controller.
// +optional
Paused bool `json:"paused,omitempty"`
// TODO: remove optional after defaulting logic introduced
// GreatDB cluster version
// +optional
Version string `json:"version"`
// Affinity of GreatDB cluster Pods
// +optional
Affinity *corev1.Affinity `json:"affinity,omitempty"`
// Persistent volume reclaim policy applied to the PVs that consumed by GreatDB cluster
// +kubebuilder:default=Retain
PVReclaimPolicy *corev1.PersistentVolumeReclaimPolicy `json:"pvReclaimPolicy,omitempty"`
// ImagePullPolicy of GreatDB cluster Pods
// +kubebuilder:default=IfNotPresent
ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
// ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images.
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
// +kubebuilder:validation:Enum=InPlace,RollingUpdate
// +kubebuilder:default=InPlace
ConfigUpdateStrategy ConfigUpdateStrategy `json:"configUpdateStrategy,omitempty"`
// Whether enable PVC reclaim for orphan PVC left by statefulset scale-in
// Optional: Defaults to false
// +optional
EnablePVReclaim *bool `json:"enablePVReclaim,omitempty"`
// Whether enable the TLS connection between GreatDB server components
// Optional: Defaults to nil
// +optional
TLSCluster *TLSCluster `json:"tlsCluster,omitempty"`
// Whether Hostnetwork is enabled for GreatDB cluster Pods
// Optional: Defaults to false
// +optional
HostNetwork *bool `json:"hostNetwork,omitempty"`
// Base node selectors of GreatDB cluster Pods, components may add or override selectors upon this respectively
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// Base annotations of GreatDB cluster Pods, components may add or override selectors upon this respectively
// +optional
Annotations map[string]string `json:"annotations,omitempty"`
// Time zone of GreatDB cluster Pods
// Optional: Defaults to UTC
// +optional
Timezone string `json:"timezone,omitempty"`
// EnableDynamicConfiguration indicates whether DynamicConfiguration is enabled for the GreatDBCluster
// +optional
EnableDynamicConfiguration *bool `json:"enableDynamicConfiguration,omitempty"`
// ClusterDomain is the Kubernetes Cluster Domain of GreatDB cluster
// Optional: Defaults to ""
// +optional
ClusterDomain string `json:"clusterDomain,omitempty"`
// StatefulSetUpdateStrategy of GreatDB cluster StatefulSets
// +optional
StatefulSetUpdateStrategy apps.StatefulSetUpdateStrategyType `json:"statefulSetUpdateStrategy,omitempty"`
}Status(GreatDbClusterStatus)用来暴露operator关心的集群的当前状态，operator从k8s apiserver获取该cluster资源对象的status来监控集群的状态。当集群的Spec（GreatDBClusterSpec）发生变化，比如用户修改了replica字段的值，operator将调动k8s资源使得集群status趋近于期望spec，具体表现为service/statefulset/pod等k8s内置资源的增删更新的操作（operator本质上也是调用k8s内置的controller）。
// GreatDBClusterStatus represents the current status of a GreatDb cluster.
type GreatDBClusterStatus struct {
ClusterID string `json:"clusterID,omitempty"`
SQLNode SQLNodeStatus `json:"sqlnode,omitemply"`
DataNode DataNodeSatatus `json:"datanode,omitempty"`
Shard ShardStatus `json:"datanode,omitempty"`
Monitor *GreatDBMonitorRef `json:"monitor,omitempty"`
//Conditions []GreatDBClusterCondition `json:"conditions,omitempty"`
}greatdb-operator需要实现cluster资源的控制逻辑，即实现接口
/home/gopath/src/great/greatdb-operator/controllers/greatdb/greatdb_cluster_control.go
package greatdb
import (
"greatdb-operator/apis/wu123.com/v1alpha1"
"greatdb-operator/apis/wu123.com/v1alpha1/defaulting"
"greatdb-operator/controller"
"greatdb-operator/reconciler"
apiequality "k8s.io/apimachinery/pkg/api/equality"
errorutils "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/tools/record"
)
// GreatDBControlInterface implements the control logic for updating GreatDBClusters and their children StatefulSets.
// It is implemented as an interface to allow for extensions that provide different semantics.
// Currently, there is only one implementation.
type ControlInterface interface {
// UpdateCluster implements the control logic for StatefulSet creation, update, and deletion
UpdateGreatDBCluster(*v1alpha1.GreatDBCluster) error
}
// NewDefaultGreatDBClusterControl returns a new instance of the default implementation GreatDBClusterControlInterface that
// implements the documented semantics for GreatDBClusters.
func NewDefaultGreatDBClusterControl(
tcControl controller.GreatDBClusterControlInterface,
sqlnodeReconciler reconciler.Reconciler,
datanodeReconciler reconciler.Reconciler,
shardReconciler reconciler.Reconciler,
//reclaimPolicyReconciler reconciler.Reconciler,
metaReconciler reconciler.Reconciler,
//orphanPodsCleaner member.OrphanPodsCleaner,
//pvcCleaner member.PVCCleanerInterface,
//pvcResizer member.PVCResizerInterface,
//pumpMemberManager reconciler.Reconciler,
//tiflashMemberManager reconciler.Reconciler,
//ticdcMemberManager reconciler.Reconciler,
//discoveryManager member.GreatDBDiscoveryManager,
GreatDBClusterStatusReconciler reconciler.Reconciler,
//conditionUpdater GreatDBClusterConditionUpdater,
recorder record.EventRecorder) ControlInterface {
return &defaultGreatDBClusterControl{
tcControl: tcControl,
sqlnodeMemberReconciler: sqlnodeReconciler,
datanodeMemberReconciler: datanodeReconciler,
shardMemberReconciler: shardReconciler,
//reclaimPolicyReconciler: reclaimPolicyReconciler,
metaReconciler: metaReconciler,
//orphanPodsCleaner: orphanPodsCleaner,
//pvcCleaner: pvcCleaner,
//pvcResizer: pvcResizer,
//pumpMemberManager: pumpMemberManager,
//tiflashMemberManager: tiflashMemberManager,
//ticdcMemberManager: ticdcMemberManager,
//discoveryManager: discoveryManager,
GreatDBClusterStatusReconciler: GreatDBClusterStatusReconciler,
//conditionUpdater: conditionUpdater,
recorder: recorder,
}
}
type defaultGreatDBClusterControl struct {
tcControl controller.GreatDBClusterControlInterface
sqlnodeMemberReconciler reconciler.Reconciler
datanodeMemberReconciler reconciler.Reconciler
shardMemberReconciler reconciler.Reconciler
//reclaimPolicyReconciler reconciler.Reconciler
metaReconciler reconciler.Reconciler
//orphanPodsCleaner member.OrphanPodsCleaner
//pvcCleaner member.PVCCleanerInterface
//pvcResizer member.PVCResizerInterface
//discoveryManager member.GreatDBDiscoveryManager
GreatDBClusterStatusReconciler reconciler.Reconciler
//conditionUpdater GreatDBClusterConditionUpdater
recorder record.EventRecorder
}
func (c *defaultGreatDBClusterControl) UpdateGreatDBCluster(dc *v1alpha1.GreatDBCluster) error {
c.defaulting(dc)
var errs []error
oldStatus := dc.Status.DeepCopy()
if err := c.updateGreatDBCluster(dc); err != nil {
errs = append(errs, err)
}
//if err := c.conditionUpdater.Update(dc); err != nil {
// errs = append(errs, err)
//}
if apiequality.Semantic.DeepEqual(&dc.Status, oldStatus) {
return errorutils.NewAggregate(errs)
}
//if _, err := c.tcControl.UpdateGreatDBCluster(dc.DeepCopy(), &dc.Status, oldStatus); err != nil {
// errs = append(errs, err)
//}
return errorutils.NewAggregate(errs)
}
func (c *defaultGreatDBClusterControl) defaulting(dc *v1alpha1.GreatDBCluster) {
defaulting.SetGreatDBClusterDefault(dc)
}
func (c *defaultGreatDBClusterControl) updateGreatDBCluster(dc *v1alpha1.GreatDBCluster) error {
// syncing all PVs managed by operator's reclaim policy to Retain
//if err := c.reclaimPolicyManager.Sync(tc); err != nil {
// return err
//}
//cleaning all orphan pods(pd, MySQL or tiflash which don't have a related PVC) managed by operator
/*
skipReasons, err := c.orphanPodsCleaner.Clean(tc)
if err != nil {
return err
}
if klog.V(10) {
for podName, reason := range skipReasons {
klog.Infof("pod %s of cluster %s/%s is skipped, reason %q", podName, tc.Namespace, tc.Name, reason)
}
}
*/
// reconcile GreatDB discovery service
//if err := c.discoveryManager.Reconcile(tc); err != nil {
// return err
//}
// works that should do to making the pd cluster current state match the desired state:
// - create or update the pd service
// - create or update the pd headless service
// - create the pd statefulset
// - sync pd cluster status from pd to GreatDBCluster object
// - set two annotations to the first pd member:
// - label.Bootstrapping
// - label.Replicas
// - upgrade the pd cluster
// - scale out/in the pd cluster
// - failover the pd cluster
if err := c.datanodeMemberReconciler.ReconcileGreatDB(dc); err != nil {
return err
}
// works that should do to making the MySQL cluster current state match the desired state:
// - waiting for the pd cluster available(pd cluster is in quorum)
// - create or update MySQL headless service
// - create the MySQL statefulset
// - sync MySQL cluster status from pd to GreatDBCluster object
// - set scheduler labels to MySQL stores
// - upgrade the MySQL cluster
// - scale out/in the MySQL cluster
// - failover the MySQL cluster
if err := c.sqlnodeMemberReconciler.ReconcileGreatDB(dc); err != nil {
return err
}
// works that should do to making the GreatDB cluster current state match the desired state:
// - waiting for the MySQL cluster available(at least one peer works)
// - create or update GreatDB headless service
// - create the GreatDB statefulset
// - sync GreatDB cluster status from pd to GreatDBCluster object
// - upgrade the GreatDB cluster
// - scale out/in the GreatDB cluster
// - failover the GreatDB cluster
if err := c.shardMemberReconciler.ReconcileGreatDB(dc); err != nil {
return err
}
// syncing the labels from Pod to PVC and PV, these labels include:
// - label.StoreIDLabelKey
// - label.MemberIDLabelKey
// - label.NamespaceLabelKey
if err := c.metaReconciler.ReconcileGreatDB(dc); err != nil {
return err
}
// cleaning the pod scheduling annotation for pd and MySQL
/*
pvcSkipReasons, err := c.pvcCleaner.Clean(tc)
if err != nil {
return err
}
if klog.V(10) {
for pvcName, reason := range pvcSkipReasons {
klog.Infof("pvc %s of cluster %s/%s is skipped, reason %q", pvcName, tc.Namespace, tc.Name, reason)
}
}
// resize PVC if necessary
if err := c.pvcResizer.Resize(tc); err != nil {
return err
}
*/
// syncing the some GreatDBcluster status attributes
// - sync GreatDBmonitor reference
return c.GreatDBClusterStatusReconciler.ReconcileGreatDB(dc)
}
var _ ControlInterface = &defaultGreatDBClusterControl{}
type ControlInterface interface {
// UpdateGreatdbCluster implements the control logic for StatefulSet creation, update, and deletion
UpdateGreatdbCluster(*v1alpha1.GreatdbCluster) error
}operator关注某种资源（比如GreatdbCluster,持续获取k8s apiserver推送的关于该资源对象的变更事件，更新本地缓存的资源对象，然后将对象作为参数调用UpdateGreatdbCluster方法更新集群。该方法的主要工作是根据传入的v1alpha1.GreatdbCluster参数分别调用sqlnode和datanode组件的Sync方法实现集群更新。因此sqlnode和datanode需要实现Manager接口）
// Manager implements the logic for syncing cluster.
type Manager interface {
// Sync implements the logic for syncing cluster.
Sync(*v1alpha1.GreatdbCluster) error
}datanode组件的Sync方法的主要工作包括：
1. 创建或更新datanode server
2. 创建或更新datanode headless service
3. 创建datanode statefulset
4. 同步datanode的DataNodeStatus
// 5. 升级datanode集群
// 6. datanode扩缩容
// 7. datanode集群故障恢复sqlnode组件的Sync方法同理：
1. 创建或更新sqlnode server
2. 创建或更新sqlnode headless service
3. 创建sqlnode statefulset
4. 同步sqlnode 的SQLNodeStatus
// 5. 升级sqlnode 集群
// 6. sqlnode 扩缩容
// 7. sqlnode 集群故障恢复shard的Sync方法主要用于集群初始化：
1. 创建或更新集群初始化Job资源
2. 创建或更新shard service
3. 同步shard 的ShardStatusPOD IP变化问题
由于K8s环境下pod重启后IP会发生变化，因此应用不应该依赖pod ip，而应该使用域名来访问pod
项目进度
当前的主要目标是实现一个初步版本，只包含集群部署与监控部署功能
目前已经实现的内容包括：
greatdb-operator自身的安装部署
在k8s 集群环境下部署一个greatdb-operator已经完成: 基于operator程序build docker镜像，然后以部署k8s deployment资源方式起一个operator pod,提供对greatdbcluster和greatdbmonitor两种自定义资源的管理
具体操作是以helm方式，执行`helm install --namespace greatdb-admin greatdb-operator greatdb-operator`命令，将在greatdb-admin namespace下部署greatdb-operator
```shell
/$ cd greatdb-operator/helm/charts && helm install --namespace greatdb-admin greatdb-operator greatdb-operator
```
执行成功可看到如下查询结果
```shell
$ kubectl get po -n greatdb-admin
NAME READY STATUS RESTARTS AGE
greatdb-controller-manager-59f7597df-4jcq6 1/1 Running 0 10s
```
此时greatdb-operator已经安装完成
部署集群的sqlnode/datanode节点
当前用户可编写GreatDBCluster资源对应的yaml文件，然后使用`kubectl apply -f xxxxxx.yaml` 部署集群所有节点
例如,有如下greatdb.yaml文件，描述了一个三个sqlnode，6个datanode的集群
```yaml
# IT IS NOT SUITABLE FOR PRODUCTION USE.
# This YAML describes a basic GreatDB cluster with minimum resource requirements,
# which should be able to run in any Kubernetes cluster with storage support.
apiVersion: wu123.com/v1alpha1
kind: GreatDBCluster
metadata:
name: cluster
spec:
version: v5.0.4
timezone: UTC
pvReclaimPolicy: Retain
enableDynamicConfiguration: true
configUpdateStrategy: RollingUpdate
sqlNode:
baseImage: greatopensource/greatdb
version: latest
replicas: 3
service:
type: NodePort
storageClassName: local-path
requests:
storage: "5Gi"
config: {}
enableMetrics: false
dataNode:
baseImage: greatopensource/greatdb
version: latest
replicas: 6
storageClassName: local-path
requests:
storage: "5Gi"
config: {}
```执行`kubectl apply -n greatdb -f greatdb.yaml`命令，成功后查询可看到所有节点pod已经部署完成
```shell
$ kubectl get po -n greatdb
NAME READY STATUS RESTARTS AGE
cluster-datanode-0 1/1 Running 0 86s
cluster-datanode-1 1/1 Running 0 86s
cluster-datanode-2 1/1 Running 0 86s
cluster-datanode-3 1/1 Running 0 86s
cluster-datanode-4 1/1 Running 0 86s
cluster-datanode-5 1/1 Running 0 86s
cluster-sqlnode-0 1/1 Running 0 86s
cluster-sqlnode-1 1/1 Running 0 86s
cluster-sqlnode-2 1/1 Running 0 86s
```查看service可看到
```shell
$ kubectl get svc -n greatdb
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
cluster-datanode-peer ClusterIP None <none> 3306/TCP 3m5s
cluster-sqlnode NodePort 10.100.148.23 <none> 3306:30690/TCP,10080:30571/TCP 3m5s
cluster-sqlnode-peer ClusterIP None <none> 10080/TCP 3m5s
```sqlnode对应的service是NodePort类型的，可以通过宿主机IP直接登陆
```shell
mysql -uroot -pgreatdb -h172.16.70.246 -P30690
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MySQL connection id is 69
Server version: 8.0.21-12-greatdbcluster5.0.4-rc GreatDB Cluster (GPL), Release rc, Revision 3d46f2558e2
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MySQL [(none)]>
```当前需要进行的工作
包括：集群监控部署和集群自动初始化两部分
其中监控功能实现设计为：在每个sqlnode/datanode 的pod中放一个mysqld_exporter容器，负责收集监控指标，同时集群部署prometheus和grafana用于拉取数据与展示
集群自动初始化功能实现设计为：用户在编写GreatDBCluster yaml文件时可以指定集群shard信息（例如有几个shard，每个shard多少个datanode),然后operator创建一个k8s Job资源负责等集群pod都起来后，连接sqlnode执行集群搭建语句
这两项工作完成之后greatdb集群的部署功能基本实现完毕
将来需要添加的功能
预计会添加
* 集群备份，扩容缩容
* 节点（pod）故障恢复
* sqlnode/datanode滚动升级
等功能，这些还需要参考其他实现。
测试test
install crd
cd ~/k8s/greatdb-operator/manifests/ && kubectl create -f greatdbcluster_crd.yaml
cd ~/k8s/greatdb-operator/manifests/ && kubectl create -f greatdbmonitor_crd.yaml安装 greatdb-operator
cd ~/k8s/greatdb-operator && docker build -t greatopensource/greatdb-operator:v0.1.1 .
cd ~/k8s/greatdb-operator/helm/charts && helm install --namespace greatdb-admin greatdb-operator
kubectl get po -n greatdb-admin
cd ~/k8s/greatdb-operator/helm/charts && helm uninstall --namespace greatdb-admin greatdb-operator部署测试集群
cd ~/k8s/greatdb-operator/examples && kubectl apply -n greatdb -f greatdb.yaml
cd ~/k8s/greatdb-operator/examples && kubectl delete -n greatdb -f greatdb.yaml
kubectl delete pvc sqlnode-cluster-sqlnode-0 sqlnode-cluster-sqlnode-1 sqlnode-cluster-sqlnode-2 -n greatdb
kubectl delete pvc datanode-cluster-datanode-0 datanode-cluster-datanode-1 datanode-cluster-datanode-2 datanode-cluster-datanode-3 datanode-cluster-datanode-4 datanode-cluster-datanode-5 -n greatdb部署监控 monitor
cd ~/k8s/greatdb-operator/docker/greatdb-monitor && docker build -t greatdb/greatdb-monitor-init:v0.0.1 .
cd ~/k8s/greatdb-operator/examples && kubectl apply -n greatdb -f monitor.yaml
cd ~/k8s/greatdb-operator/examples && kubectl delete -n greatdb -f monitor.yaml
kubectl port-forward -n greatdb svc/monitor-grafana 3000 > pf3000.out &greatdb 镜像
cd ~/k8s/greatdb-operator/docker/greatdb && docker build -t greatopensource/greatdb:latest .
docker run --name greatdb-test --network kind --hostname greatdb-host -d greatopensource/greatdb:latest
docker cp ~/k8s/greatdb-operator/docker/create_config.sh greatdb-test:/
docker exec -it greatdb-test /bin/bash
chmod +x /create_config.sh && /create_config.sh sqlnode 3306 > /var/lib/greatdb-cluster/sqlnode.cnf
cd /user/local/greatdb-cluster
bin/greatdb_init --defaults-file=/var/lib/greatdb-cluster/sqlnode.cnf --cluster-user=greatdb --cluster-host=% --cluster-password=greatdb --node-type=sqlnode
docker container stop greatdb-test && docker container rm greatdb-test
扫一扫
258
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
