#!/usr/bin/env python # -*- coding: utf-8 -*- # @Date : 2018-10-11 17:55:26 # @Author : Sheldon (thisisscret@qq.com) # @blogs : 谢耳朵的派森笔记 # @Link : https://www.cnblogs.com/shld/
import pandas as pd from joblib import Parallel, delayed def apply_parallel(df, func, n=-2): """利用 Parallel 和 delayed 函数实现并行运算,思路是把Dataframe分割喂给包含apply的函数 @params df: 需要apply的Dataframe @params func: 包含apply的函数,(不是apply的参数那个函数),需自己定义, @params n: n为线程数,默认为cpu数-1,-1为cpu数,可自定义 @return Dataframe: apply参数函数得到的Dataframe
""" if n is None: n = -1 dflength = len(df) cpunum = cpu_count() if dflength<cpunum: spnum = dflength if n<0: spnum = cpunum+n+1 else: spnum = n or 1 sp = list(range(dflength)[::int(dflength/spnum+0.5)]) sp.append(dflength) slice_gen = (slice(*idx) for idx in zip(sp[:-1],sp[1:])) results = Parallel(n_jobs=n)(delayed(func)(df[slc]) for slc in slice_gen) return pd.concat(results)