pandas,apply并行计算的一个demo

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2018-10-11 17:55:26
# @Author : Sheldon (thisisscret@qq.com)
# @blogs   : 谢耳朵的派森笔记
# @Link     : https://www.cnblogs.com/shld/

import pandas as pd
from joblib import Parallel, delayed

def apply_parallel(df, func, n=-2):
    """利用 Parallel 和 delayed 函数实现并行运算，思路是把Dataframe分割喂给包含apply的函数
       @params df:        需要apply的Dataframe
       @params func:      包含apply的函数，（不是apply的参数那个函数），需自己定义，
       @params n:         n为线程数，默认为cpu数-1，-1为cpu数，可自定义
       @return Dataframe: apply参数函数得到的Dataframe
　　 """

    if n is None:
        n = -1
    dflength = len(df)
    cpunum = cpu_count()
    if dflength<cpunum:
        spnum = dflength
    if n<0:
        spnum = cpunum+n+1
    else:
        spnum = n or 1
    
    sp = list(range(dflength)[::int(dflength/spnum+0.5)])
    sp.append(dflength)
    slice_gen = (slice(*idx) for idx in zip(sp[:-1],sp[1:]))
    results = Parallel(n_jobs=n)(delayed(func)(df[slc]) for slc in slice_gen)
    return pd.concat(results)                            

转载于:https://www.cnblogs.com/shld/p/9774180.html