6.824 lab4 Part A（分布式shard database）

阅读文档

https://pdos.csail.mit.edu/6.824/labs/lab-shard.html

浅析

lab4 的架构是典型的 M/S 架构（a configuration service and a set of replica groups)，

configuration service

1. 由若干 shardmaster 利用 raft 协议保证一致性的集群；
2. 管理 configurations 的顺序：每个 configuration 描述 replica group 以及每个 group 分别负责存储哪些 shards；
3. 响应 Join/Leave/Move/Query 请求，并且对 configuration 做出相应的改变；

JOIN：会给一组GID -> SERVER的映射。其实就是把这些GID 组，加到MASTER的管理范围里来。那么有新的GROUP来了。每台机器可以匀一些SHARD过去

LEAVE：是给一组GID，表示这组GID的SERVER机器们要走。那么他们管的SHARD又要匀给还没走的GROUP

MOVE：是指定某个SHARD 归这个GID管

QUERY：就是根据CONFIG NUM来找到对应的CONFIG里的SHARD 规则

replica group

1. 由若干 shardkv 利用 raft 协议保证一致性的集群；
2. 负责具体数据的存储（一部分），组合所有 group 的数据即为整个 database 的数据；
3. 响应对应 shards 的 Get/PutAppend 请求，并保证 linearized；
4. 周期性向 shardmaster 进行 query 获取 configuration，并且进行 migration 和 update；

Sharemaster 主要负责根据 Client 提供的分区规则，将数据储存在不同的replica group 中

Sharemaster 有多台机器组成，相互间使用 Raft 协议来保证一致性。

每一个 replica group由多台机器组成，他们之间也是通过 Raft 协议来保证一致性。

Client代码

common.go中新增参数

type JoinArgs struct {
	Servers map[int][]string // new GID -> servers mappings
	ClientId int64
	RequestId int
}

type JoinReply struct {
	WrongLeader bool
	Err         Err
}

type LeaveArgs struct {
	GIDs []int
	ClientId int64
	RequestId int
}

type LeaveReply struct {
	WrongLeader bool
	Err         Err
}

type MoveArgs struct {
	Shard int
	GID   int
	ClientId int64
	RequestId int
}

type MoveReply struct {
	WrongLeader bool
	Err         Err
}

package shardmaster

import (
    "labrpc"
)
import "time"
import "crypto/rand"
import "math/big"

const RetryInterval = time.Duration(100 * time.Millisecond)

type Clerk struct {
    servers    []*labrpc.ClientEnd
    id   int64
    seqNum int
    lastLeader   int
}

func Nrand() int64 {
    max := big.NewInt(int64(1) << 62)
    bigx, _ := rand.Int(rand.Reader, max)
    x := bigx.Int64()
    return x
}


func MakeClerk(servers []*labrpc.ClientEnd) *Clerk {
    ck := new(Clerk)
    ck.servers = servers
    ck.id = Nrand()
    ck.seqNum = 0
    ck.lastLeader = 0
    return ck
}

func (ck *Clerk) Query(num int) Config {
    args := QueryArgs{Num: num}
    for {
        var reply QueryReply
        if ck.servers[ck.lastLeader].Call("ShardMaster.Query", &args, &reply) && !reply.WrongLeader {
            return reply.Config
        }
        ck.lastLeader = (ck.lastLeader + 1) % len(ck.servers)
        time.Sleep(RetryInterval)
    }
}

func (ck *Clerk) Join(servers map[int][]string) {
    args := JoinArgs{Servers: servers, Cid:ck.id, SeqNum:ck.seqNum}
    ck.seqNum++
    for {
        var reply JoinReply
        if ck.servers[ck.lastLeader].Call("ShardMaster.Join", &args, &reply) && !reply.WrongLeader {
            return
        }
        ck.lastLeader = (ck.lastLeader + 1) % len(ck.servers)
        time.Sleep(RetryInterval)
    }
}

func (ck *Clerk) Leave(gids []int) {
    args := LeaveArgs{GIDs: gids, Cid:ck.id, SeqNum:ck.seqNum}
    ck.seqNum++
    for {
        var reply LeaveReply
        if ck.servers[ck.lastLeader].Call("ShardMaster.Leave", &args, &reply) && !reply.WrongLeader {
            return
        }
        ck.lastLeader = (ck.lastLeader + 1) % len(ck.servers)
        time.Sleep(RetryInterval)
    }
}

func (ck *Clerk) Move(shard int, gid int) {
    args := MoveArgs{Shard: shard, GID: gid, Cid:ck.id, SeqNum:ck.seqNum}
    ck.seqNum++
    for {
        var reply MoveReply
        if ck.servers[ck.lastLeader].Call("ShardMaster.Move", &args, &reply) && !reply.WrongLeader {
            return
        }
        ck.lastLeader = (ck.lastLeader + 1) % len(ck.servers)
        time.Sleep(RetryInterval)
    }
}

Server实现

package shardmaster

import (
    "log"
    "math"
    "raft"
    "time"
)
import "labrpc"
import "sync"
import "labgob"

type ShardMaster struct {
    mu      sync.Mutex
    me      int
    rf      *raft.Raft
    applyCh chan raft.ApplyMsg
    // Your data here.
    configs []Config // indexed by config num
    chMap   map[int]chan Op
    cid2Seq map[int64]int
    killCh  chan bool
}

type Op struct {
    OpType  string "operation type(eg. join/leave/move/query)"
    Args    interface{} // could be JoinArgs, LeaveArgs, MoveArgs and QueryArgs, in reply it could be config
    Cid     int64
    SeqNum  int
}

func (sm *ShardMaster) Join(args *JoinArgs, reply *JoinReply) {
    originOp := Op{"Join",*args,args.Cid,args.SeqNum}
    reply.WrongLeader = sm.templateHandler(originOp)
}

func (sm *ShardMaster) Leave(args *LeaveArgs, reply *LeaveReply) {
    originOp := Op{"Leave",*args,args.Cid,args.SeqNum}
    reply.WrongLeader = sm.templateHandler(originOp)
}

func (sm *ShardMaster) Move(args *MoveArgs, reply *MoveReply) {
    originOp := Op{"Move",*args,args.Cid,args.SeqNum}
    reply.WrongLeader = sm.templateHandler(originOp)
}

func (sm *ShardMaster) Query(args *QueryArgs, reply *QueryReply) {
    reply.WrongLeader = true;
    originOp := Op{"Query",*args,Nrand(),-1}
    reply.WrongLeader = sm.templateHandler(originOp)
    if !reply.WrongLeader {
        sm.mu.Lock()
        defer sm.mu.Unlock()
        if args.Num >= 0 && args.Num < len(sm.configs) {
            reply.Config = sm.configs[args.Num]
        } else {
            reply.Config = sm.configs[len(sm.configs) - 1]
        }
    }
}

func (sm *ShardMaster) templateHandler(originOp Op) bool {
    wrongLeader := true
    index,_,isLeader := sm.rf.Start(originOp)
    if !isLeader {return wrongLeader}
    ch := sm.getCh(index,true)
    op := sm.beNotified(ch,index)
    if equalOp(op,originOp) {
        wrongLeader = false
    }
    return wrongLeader
}

func (sm *ShardMaster) beNotified(ch chan Op, index int) Op {
    select {
    case notifyArg := <- ch :
        close(ch)
        sm.mu.Lock()
        delete(sm.chMap,index)
        sm.mu.Unlock()
        return notifyArg
    case <- time.After(time.Duration(600)*time.Millisecond):
        return Op{}
    }
}

func equalOp(a Op, b Op) bool{
    return a.SeqNum == b.SeqNum && a.Cid == b.Cid && a.OpType == b.OpType
}

func (sm *ShardMaster) Kill() {
    sm.rf.Kill()
    sm.killCh <- true
}
// needed by shardkv tester
func (sm *ShardMaster) Raft() *raft.Raft {
    return sm.rf
}

func (sm *ShardMaster) getCh(idx int, createIfNotExists bool) chan Op{
    sm.mu.Lock()
    defer sm.mu.Unlock()
    if _, ok := sm.chMap[idx]; !ok {
        if !createIfNotExists {return nil}
        sm.chMap[idx] = make(chan Op,1)
    }
    return sm.chMap[idx]
}

func (sm *ShardMaster) updateConfig(op string, arg interface{}) {
    cfg := sm.createNextConfig()
    if op == "Move" {
        moveArg := arg.(MoveArgs)
        if _,exists := cfg.Groups[moveArg.GID]; exists {
            cfg.Shards[moveArg.Shard] = moveArg.GID
        } else {return}
    }else if op == "Join" {
        joinArg := arg.(JoinArgs)
        for gid,servers := range joinArg.Servers {
            newServers := make([]string, len(servers))
            copy(newServers, servers)
            cfg.Groups[gid] = newServers
            sm.rebalance(&cfg,op,gid)
        }
    } else if op == "Leave"{
        leaveArg := arg.(LeaveArgs)
        for _,gid := range leaveArg.GIDs {
            delete(cfg.Groups,gid)
            sm.rebalance(&cfg,op,gid)
        }
    } else {
        log.Fatal("invalid area",op)
    }
    sm.configs = append(sm.configs,cfg)
}

func (sm *ShardMaster) createNextConfig() Config {
    lastCfg := sm.configs[len(sm.configs)-1]
    nextCfg := Config{Num: lastCfg.Num + 1, Shards: lastCfg.Shards, Groups: make(map[int][]string)}
    for gid, servers := range lastCfg.Groups {
        nextCfg.Groups[gid] = append([]string{}, servers...)
    }
    return nextCfg
}

func (sm *ShardMaster) rebalance(cfg *Config, request string, gid int) {
    shardsCount := sm.groupByGid(cfg) // gid -> shards
    switch request {
    case "Join":
        avg := NShards / len(cfg.Groups)
        for i := 0; i < avg; i++ {
            maxGid := sm.getMaxShardGid(shardsCount)
            cfg.Shards[shardsCount[maxGid][0]] = gid
            shardsCount[maxGid] = shardsCount[maxGid][1:]
        }
    case "Leave":
        shardsArray,exists := shardsCount[gid]
        if !exists {return}
        delete(shardsCount,gid)
        if len(cfg.Groups) == 0 { // remove all gid
            cfg.Shards = [NShards]int{}
            return
        }
        for _,v := range shardsArray {
            minGid := sm.getMinShardGid(shardsCount)
            cfg.Shards[v] = minGid
            shardsCount[minGid] = append(shardsCount[minGid], v)
        }
    }
}
func (sm *ShardMaster) groupByGid(cfg *Config) map[int][]int {
    shardsCount := map[int][]int{}
    for k,_ := range cfg.Groups {
        shardsCount[k] = []int{}
    }
    for k, v := range cfg.Shards {
        shardsCount[v] = append(shardsCount[v], k)
    }
    return shardsCount
}
func (sm *ShardMaster) getMaxShardGid(shardsCount map[int][]int) int {
    max := -1
    var gid int
    for k, v := range shardsCount {
        if max < len(v) {
            max = len(v)
            gid = k
        }
    }
    return gid
}
func (sm *ShardMaster) getMinShardGid(shardsCount map[int][]int) int {
    min := math.MaxInt32
    var gid int
    for k, v := range shardsCount {
        if min > len(v) {
            min = len(v)
            gid = k
        }
    }
    return gid
}
func send(notifyCh chan Op,op Op) {
    notifyCh <- op
}
func StartServer(servers []*labrpc.ClientEnd, me int, persister *raft.Persister) *ShardMaster {
    sm := new(ShardMaster)
    sm.me = me
    sm.configs = make([]Config, 1)
    sm.configs[0].Groups = map[int][]string{}
    labgob.Register(Op{})
    labgob.Register(JoinArgs{})
    labgob.Register(LeaveArgs{})
    labgob.Register(MoveArgs{})
    labgob.Register(QueryArgs{})
    sm.applyCh = make(chan raft.ApplyMsg)
    sm.rf = raft.Make(servers, me, persister, sm.applyCh)
    // Your code here.
    sm.chMap = make(map[int]chan Op)
    sm.cid2Seq = make(map[int64]int)
    sm.killCh = make(chan bool,1)
    go func() {
        for {
            select {
            case <-sm.killCh:
                return
            case applyMsg := <-sm.applyCh:
                if !applyMsg.CommandValid {continue}
                op := applyMsg.Command.(Op)
                sm.mu.Lock()
                maxSeq,found := sm.cid2Seq[op.Cid]
                if op.SeqNum >= 0 && (!found || op.SeqNum > maxSeq) {
                    sm.updateConfig(op.OpType,op.Args)
                    sm.cid2Seq[op.Cid] = op.SeqNum
                }
                sm.mu.Unlock()
                if notifyCh := sm.getCh(applyMsg.CommandIndex,false); notifyCh != nil {
                    send(notifyCh,op)
                }
            }
        }
    }()
    return sm
}