Part III: Distributing MapReduce tasks
任务描述
完成单机分布式的MapReduce任务。
完成mapreduce/schedule.go
中的schedule()
函数。
schedule()
函数的功能是将任务交付给可用的worker()。
涉及到的代码
mapreduce/master.go
mapreduce/common_rpc.go
mapreduce/worker.go
Hint
-
go test -race -run TestParallel > out
.测试有没有竞争条件。 -
go test -run TestParallel
-
sync.WaitGroup
Sequential的schedule调度函数
func(phase jobPhase) {
switch phase {
case mapPhase:
for i, f := range mr.files {
doMap(mr.jobName, i, f, mr.nReduce, mapF)
}
case reducePhase:
for i := 0; i < mr.nReduce; i++ {
doReduce(mr.jobName, i, mergeName(mr.jobName, i), len(mr.files), reduceF)
}
}
}
设计思路
使用一个变量currentTask
来记录当前需要被分配的任务id,当currentTask==ntasks
时,代表任务已经分配完成。
主goroutine的任务是,使用非阻塞IO不断地读registerChan
里的内容,每侦听到一个就建立一个新的goroutine来做RPC调用。
原子锁的使用
var ops uint64 = 0
atomic.AddUint64(&ops, 1)
atomic.LoadUint64(&ops)
使用原子锁来记录当前的任务编号。
代码
func schedule(jobName string, mapFiles []string, nReduce int, phase jobPhase, registerChan chan string) {
var ntasks int
var n_other int // number of inputs (for reduce) or outputs (for map)
switch phase {
case mapPhase:
ntasks = len(mapFiles)
n_other = nReduce
case reducePhase:
ntasks = nReduce
n_other = len(mapFiles)
}
fmt.Printf("Schedule: %v %v tasks (%d I/Os)\n", ntasks, phase, n_other)
// All ntasks tasks have to be scheduled on workers. Once all tasks
// have completed successfully, schedule() should return.
//
// Your code here (Part III, Part IV).
//
var currentTask int32= 0
var mutexForTask = &sync.Mutex{}
var wg sync.WaitGroup
for atomic.LoadInt32(¤tTask)<int32(ntasks){
select{
case workerName := <-registerChan:
wg.Add(1)
go func(routeWorker string) {
for{
mutexForTask.Lock()
currentId := atomic.LoadInt32(¤tTask)
if currentId<int32(ntasks){
atomic.AddInt32(¤tTask,1)
mutexForTask.Unlock()
//start do work
args:= new(DoTaskArgs)
args.File = mapFiles[currentId]
args.JobName = jobName
args.NumOtherPhase = n_other
args.Phase = phase
args.TaskNumber = int(currentId)
ok := call(routeWorker, "Worker.DoTask", args, new(struct{}))
if ok == false {
debug("rpc error currentId ",currentId)
} else {
debug("success ",currentId)
}
}
if currentId >= int32(ntasks){
wg.Add(-1)
mutexForTask.Unlock()
debug("work done ",currentId)
return
}
}
}(workerName)
default:
continue
}
}
//wait for job complete
wg.Wait()
fmt.Printf("Schedule: %v done\n", phase)
}