1. 学习资料
MIT 6.824 课程地址:http://nil.csail.mit.edu/6.824/2020/schedule.html
Map Reduce 论文:http://nil.csail.mit.edu/6.824/2020/papers/mapreduce.pdf
理解Map Reduce时参考的一些文章:
https://highlyscalable.wordpress.com/2012/02/01/mapreduce-patterns/
http://stevekrenzel.com/finding-friends-with-mapreduce
https://www.journaldev.com/8848/mapreduce-algorithm-example
2. MR作业代码思路
1. 题目主要是让模拟 一个master节点 调度多个 worker 节点的情况,刚开始拿到这个题目有些无从下手,根据课程提示:
One way to get started is to modify mr/worker.go's Worker() to send an RPC to the master asking for a task. Then modify the master to respond with the file name of an as-yet-unstarted map task. Then modify the worker to read that file and call the application Map function, as in mrsequential.go
2.首先从worker.go 中的Worker() 发出RPC请求着手,在worker节点中,worker通过一个for循环不停的向master节点发送RPC请求,master则进行任务分配。
// main/mrworker.go calls this function.
//
const (
MAPTASK=iota
REDUCETASK
FinishTask
)
// 分配任务时的 入参
type TaskArgs struct {
}
// 分配任务时的返回参数
type TaskReply struct {
TaskType int //返回的任务类型 标记是类型为三种 map任务 reduce任务 任务结束
TaskName string //任务名称 map任务名为:输入的文件名 reduce任务名称为0 到(NReduce-1)
NReduce int
}
func Worker(mapf func(string, string) []KeyValue,
reducef func(string, []string) string) {
// Your worker implementation here.
// uncomment to send the Example RPC to the master.
// CallExample()
//TODO send rpc to master call for a task
//开始 Work
for {
taskArgs := new(TaskArgs)
taskReply := new(TaskReply)
//远程调用 master的AssignTask方法,取得任务
e := call("Master.AssignTask", taskArgs, taskReply)
if e == false {
fmt.Println("assign task err"+taskReply.TaskName)
continue
}
//根据返回的task类型 分别执行 map任务或reduce任务,若全部执行完成则退出
switch taskReply.TaskType {
case MAPTASK:
doMap(taskReply.TaskName, mapf, taskReply.NReduce)
case REDUCETASK:
doReduce(taskReply.TaskName, reducef)
case FinishTask:
os.Exit(1)
}
continue
}
}
3.master端,首先需要得到待处理的任务,才可以进行任务分配,在MakeMaster方法中初始化Master实例
type Master struct {
// Your definitions here.
AllFilesName map[string]int //需要计算的文件名称(需要map的任务),key为文件名称,value为状态 INIT , PROCESS,FINISH
ReduceTaskState map[string]int //reduce 任务列表 key 为任务号 (由nReduce数量指定) ,value 为任务状态 INIT , PROCESS,FINISH
NReduce int // n reduce task
MapFinished bool //标记map是否全部完成
ReduceFinished bool //标记reduce是否全部完成
RWLock *sync.RWMutex
}
// create a Master.
// main/mrmaster.go calls this function.
// nReduce is the number of reduce tasks to use.
//
func MakeMaster(files []string, nReduce int) *Master {
m := Master{
AllFilesName: make(map[string]int),
ReduceTaskState: make(map[string]int),
RWLock: new(sync.RWMutex),
}
// Your code here.
//
for _, v := range files {
m.AllFilesName[v] = INIT
}
for i := 0; i < nReduce; i++ {
m.ReduceTaskState[strconv.Itoa(i)] = INIT
}
m.NReduce = nReduce
m.MapFinished = false
m.ReduceFinished = false
m.server()
return &m
}
4.master端会持续调用Done()方法来检查map与reduce方法的执行情况,若全部执行完成则退出
var once sync.Once
//
// main/mrmaster.go calls Done() periodically to find out
// if the entire job has finished.
//
func (m *Master) Done() bool {
// Your code here.
if m.MapFinished == false {
m.MapFinished = m.isFinishMap() //遍历map task 确定map任务是否全部完成
}
if m.MapFinished == false {
for k, v := range m.AllFilesName {
if v == INIT { //若map task 为INIT状态,则传入 map task channel
MapTaskCh <- k
return false
}
}
return false
}
//若map task 全部完成,则关闭map task channel
once.Do(func() {
close(MapTaskCh)
})
m.ReduceFinished = m.isFinishReduce() //遍历reduce task 确定reduce任务是否全部完成
if m.ReduceFinished == false {
for k, v := range m.ReduceTaskState {
if v == INIT {
ReduceTaskCh <- k //若reduce task 为INIT状态,则传入 reduce task channel
return false
}
}
return false
}
m.ReduceFinished=true
close(ReduceTaskCh)
return true
}
5.master 端根据channel中传入的任务来分配任务至worker端
//接受worker rpc请求并分配任务
func (m *Master) AssignTask(args *TaskArgs, reply *TaskReply) error {
//若map任务未全部完成
if m.MapFinished == false {
reply.TaskType = MAPTASK
reply.TaskName = <-MapTaskCh //从map task channel中领取任务
reply.NReduce = m.NReduce
go m.TimeClocker(reply.TaskName, MAPTASK) //计时十秒,若任务状态不变更,则重新将任务传入待处理的map task channel中
return nil
}
if (m.ReduceFinished == false) && (m.MapFinished==true) {
reply.TaskType = REDUCETASK
reply.TaskName = <-ReduceTaskCh //从reduce task channel中领取任务
reply.NReduce = m.NReduce
go m.TimeClocker(reply.TaskName, REDUCETASK) //计时十秒,若任务状态不变更,则重新将任务传入待处理的reduce task channel中
return nil
}
if m.MapFinished &&m.ReduceFinished{ 处理完成,worker可退出
reply.TaskType = FinishTask
return nil
}
return nil
}
6. worker端接受分配任务,若任务为map类型则做map操作
//用于返回task的处理结果
type TaskStatusArgs struct {
TaskType int
TaskName string
TaskStatus int
}
type TaskStatusReply struct {
}
func doMap(fileName string, mapf func(string, string) []KeyValue, nReduce int) {
taskStatusArgs := TaskStatusArgs{
TaskType: MAPTASK, //返回处理结果状态
TaskName: fileName, //返回任务名
}
taskStatusReply := TaskStatusReply{}
file, err := os.Open(fileName)
defer file.Close()
// 若出错,则返回给master端 处理错误,并重置任务状态
if err != nil {
fmt.Println(err)
taskStatusArgs.TaskStatus = PROCFAIL
e := call("Master.ProcessStatus", &taskStatusArgs, &taskStatusReply)
if e == false {
fmt.Println(fileName + " call ProcessStatus error")
}
return
}
content, err := ioutil.ReadAll(file)
// 若出错,则返回给master端 处理错误,并重置任务状态
if err != nil {
fmt.Println(err)
taskStatusArgs.TaskStatus = PROCFAIL
e := call("Master.ProcessStatus", &taskStatusArgs, &taskStatusReply)
if e == false {
fmt.Println(fileName + " call ProcessStatus error")
}
return
}
intermediate := []KeyValue{}
kva := mapf(fileName, string(content)) //将读取的文件内容做map操作
partition(kva,fileName,nReduce) //将map生成的结果内容分割
taskStatusArgs.TaskStatus=PROCSUC
e := call("Master.ProcessStatus", &taskStatusArgs, &taskStatusReply)
if e == false {
fmt.Println(fileName + " call ProcessStatus error")
}
return
}
7.说一下这个partition函数,我在做的时候这块开始有点疑问,没太懂map与reduce之间是如何进行交互的。我的理解是对文本做完map后
如 abc.txt 内容做完map操作生成intermediate为[{"a":1},{"b":"1"},{"c":"1"},{"a":1}],对intermediate进行partition, 若NReduce=2 则我们对intermediate 中的key值做ihash操作,生成一个int 类型的k值,并对其取余 有 k%NReduce。这样我们就得到了一个0 -(NReduce)之间的数字与Reduce任务号对应,根据这个数字我们可以把key存入不同的文件中,并且这个数字将intermediate 分成了NReduce 份。这样经过partition后,我们得到了名为 mr-abc.txt-0 mr-abc.txt-1的两个文件,来供Reduce读取
func partition(intermediate []KeyValue,filename string, nReduce int) {
intermediateMap:=make(map[string][]KeyValue)
for _, kva := range intermediate {
reduceTaskNum:=strconv.Itoa(ihash(kva.Key)%nReduce)
intermediateMap[reduceTaskNum]=append(intermediateMap[reduceTaskNum],kva)
}
for reduceNum,kva:=range intermediateMap{
partitionFileName:=fmt.Sprintf("mr-%v-%v",filename,reduceNum)
ofile,_:=os.OpenFile(partitionFileName,os.O_WRONLY|os.O_CREATE|os.O_TRUNC,os.ModePerm)
defer ofile.Close()
enc := json.NewEncoder(ofile)
for _,kv:=range kva{
err := enc.Encode(&kv)
if err != nil {
fmt.Print(err)
}
}
}
}
8.worker端接受任务,若任务为reduce类型则做reduce操作,其中reduce任务号为0-(NReduce-1)的int数字,我们根据获得的数字来读取相应需要处理的任务,如得到0 则处理名称为 “^mr-.*-0$”(正则表达式)的文件
3.遇到的问题
问题1:从git上拉取的code没办法直接运行报找不到module
我的原因是本地将GO111MODULE打开了,项目中拉取的代码import包时使用了相对路径,把GO111MODULE置为off或把相对路径更改
4.本文章旨在交流经验思路,自己学习mapreduce时间不长,思路若有问题请指出,谢谢