python实现简易版本mapReduce

定义Cal类，通过map实现归类，reduce实现聚合。最后通过多线程演示了mapReduce计算过程。

from threading import Thread


class Cal(object):
    def __init__(self, start, end):
        self.result = 0
        self.start = start
        self.end = end

    def map(self):
        for i in range(self.start, self.end):
            self.result += i

    def reduce(self, other):
        self.result += other.result


def generate_worker(data):
    for index in range(1, len(data)):
        cal = Cal(data[index-1], data[index])
        yield cal

def generate_threads(workers):
    for worker in workers:
        thread = Thread(target=worker.map)
        thread.start()
        yield thread


def main():
    # [1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101]
    data = [item for item in range(1, 102, 10)]

    workers = list(generate_worker(data))
    threads = list(generate_threads(workers))

    for thread in threads:
        thread.join()

    start = workers[0]
    for worker in workers[1:]:
        start.reduce(worker)

    assert start.result == 5050


if __name__ == '__main__':
    main()