初学Python-threadpool多线程编程

1、示例及说明
/Users/nisj/PycharmProjects/EsDataProc/thread_pool_ping.py

# -*- coding=utf-8 -*-
import threadpool
import time
import urllib2

urls = [
    'http://www.163.com',
    'http://www.sohu.com',
    'http://www.ebay.com',
    'http://www.sina.com.cn',
    'http://www.baidu.com'
]

def myRequest(url):
    resp = urllib2.urlopen(url)
    print url, resp.getcode()


def timeCost(request, n):
  print "Elapsed time: %s" % (time.time()-start)

start = time.time()
pool = threadpool.ThreadPool(5)
reqs = threadpool.makeRequests(myRequest, urls, timeCost)
[ pool.putRequest(req) for req in reqs ]
pool.wait()

解释关键代码：
ThreadPool(poolsize)  
表示最多可以创建poolsize这么多线程；
makeRequests(some_callable, list_of_args, callback)  
makeRequests创建了要开启多线程的函数，以及函数相关参数和回调函数，其中回调函数可以不写，default是无，也就是说makeRequests只需要2个参数就可以运行。

2、计算DAU的代码用多线程控制
/Users/nisj/PycharmProjects/EsDataProc/dau-threadpool.py

# -*- coding=utf-8 -*-
import warnings
import datetime
import time
import os
import re
import threadpool
from itertools import groupby
from operator import itemgetter

warnings.filterwarnings("ignore")

today = datetime.date.today()
yesterday = today - datetime.timedelta(days=1)
tomorrow = today + datetime.timedelta(days=1)

# batch_date = today - datetime.timedelta(days=52)

now_time = time.strftime('%Y-%m-%d %X', time.localtime())
print "当前时间是：",now_time

def user_dau_proc(batch_date):
    os.system("""/usr/bin/mysql -hMysqlIp -P3306 -uMysqlUser -pMysqlPasswd -e "use funnyai_data; \
                delete from xx_user_dau_static where data_date='%s'; \
                 " """ % (batch_date))

    def dau_data_proc(dau_data, data_type):
        dau_data = set(dau_data)
        dau_data = list(dau_data)
        dau_list = []
        for dd_list in dau_data:
            ddl = re.split('\t', dd_list.replace('\n', ''))
            dau_list.append(ddl)

        dau_data_sorted = sorted(dau_list, key=itemgetter(0, 1))
        groupby_dau_data = groupby(dau_data_sorted, key=itemgetter(0, 1))


        rl = []
        for key, item in groupby_dau_data:
            item_cnt = 0
            for jtem in item:
                item_cnt += 1
                groupby_dau_list = (key, item_cnt)
            rl.append(groupby_dau_list)

        for x in rl:
            appsource = x[0][0]
            appkey = x[0][1]
            data_type = data_type
            data_cnt = x[1]
            etl_time = time.strftime('%Y-%m-%d %X', time.localtime())

            os.system("""/usr/bin/mysql -hMysqlIp -P3306 -uMysqlUser -pMysqlPasswd -e "use funnyai_data; \
                        insert into xx_user_dau_static(data_date,appsource,appkey,data_type,data_cnt,etl_time) \
                        select '%s','%s','%s','%s','%s','%s'; \
                         " """ % (batch_date, appsource, appkey, data_type, data_cnt, etl_time))

    ytd_dau_data = os.popen("""/usr/lib/hive-current/bin/hive -e " \
            select appsource,appkey,identifier \
            from bi_all_access_log \
            where pt_day = '%s' " \
            """
                        % (batch_date)).readlines();


    dau_data_proc(ytd_dau_data, 'dau');


'''
主程序多线程调用
'''
batch_date = ['2016-10-01', '2016-10-02', '2016-10-03', '2016-10-04', '2016-10-05', '2016-10-06', '2016-10-07', '2016-10-08']
requests = threadpool.makeRequests(user_dau_proc, batch_date)
main_pool = threadpool.ThreadPool(8)
[main_pool.putRequest(req) for req in requests]
if __name__ == '__main__':
    while True:
        try:
            time.sleep(1)
            main_pool.poll()
        except KeyboardInterrupt:
            print("**** Interrupted!")
            break
        except threadpool.NoResultsPending:
            break

    if main_pool.dismissedWorkers:
        print("Joining all dismissed worker threads...")
        main_pool.joinAllDismissedWorkers()


now_time = time.strftime('%Y-%m-%d %X', time.localtime())
print "当前时间是：",now_time

3、个人的理解
所谓多线程，就是计算机同时并行处理多个事情。而这有个前提条件，并非所有的情形都适合多线程，仅适用于没有相互依赖可并行处理的程序可使用多线程来控制；否则，可能达不到预期的效果。