任务分发框架 gearmand

安装

yum -y install gearmand
pip install gearman

---

Worker 脚本

gearman运行需要三部分：client,job,worker

client负责任务请求，job负责任务分发，worker负责任务执行

---

启动job

gearmand –log-file gearmand.log –listen 115.28.6.1 –port=4730 –verbose=INFO

启动worker

python worker.py

启动client

python client.py

---

worker 脚本 worker.py

import gearman

def task_listener_reverse(gearman_worker, gearman_job):
    '''reverse string'''
    print 'task_listener_reverse:'
    print gearman_job
    return gearman_job.data[::-1] 


if __name__ == '__main__':
    print 'Workered'
    gm_worker = gearman.GearmanWorker(['115.28.6.1:4730'])
    gm_worker.register_task('reverse', task_listener_reverse) # reverse is task name,it must be same as client task name
    gm_worker.work()

---

client 脚本 client.py

from gearman import GearmanClient

def check_request_status(job_request):
    if job_request.complete:
        print "Job %s finished!  Result: %s - %s" % (job_request.job.unique, job_request.state, job_request.result)
    elif job_request.timed_out:
        print "Job %s timed out!" % job_request.unique
    elif job_request.state == 'JOB_UNKNOWN':
        print "Job %s connection failed!" % job_request.unique


if __name__ == '__main__':
    gm_client = GearmanClient(['115.28.6.1:4730'])
    current_request = gm_client.submit_job('reverse', 'arbitrary binary data', background=False, wait_until_complete=True)

    check_request_status(current_request)

---

client 脚本2 分发多个任务 client2.py

from gearman import GearmanClient

def check_request_status(job_request):
    if job_request.complete:
        print "Job %s finished!  Result: %s - %s" % (job_request.job.unique, job_request.state, job_request.result)
    elif job_request.timed_out:
        print "Job %s timed out!" % job_request.unique
    elif job_request.state == 'JOB_UNKNOWN':
        print "Job %s connection failed!" % job_request.unique


if __name__ == '__main__':
    gm_client = GearmanClient(['115.28.6.1:4730'])
    list_of_jobs = [dict(task="reverse", data="binary data"), dict(task="reverse", data="other binary data")]
    submitted_requests = gm_client.submit_multiple_jobs(list_of_jobs, background=False, wait_until_complete=False)
    completed_requests = gm_client.wait_until_jobs_completed(submitted_requests, poll_timeout=5.0)

    for completed_job_request in completed_requests:
        check_request_status(completed_job_request)

---

官方文档
http://pythonhosted.org//gearman/library.html