agg函数加入进度条

在panda之中，我们经常需要使用到agg函数来将pandas之中的某几列聚合起来

from tqdm import tqdm
import pandas as pd
import numpy as np
df = pd.DataFrame({"A":[1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0], 
                   "B":[1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0], 
                   "C":[1.0, 1.5, 2.0, 2.0, 3.0, 4.0, 5.0, 6.0, 10.0],
                   "D":[2.0, 5.0, 3.0, 6.0, 4.0, 2.0, 5.0, 1.0, 2.0],
                   "E":['a', 'a', 'b', 'a', 'b', 'b', 'b', 'a', 'a']}) 

df2 = df.groupby('B').agg({
                           'C': 'mean',
                           'D': 'sum',
                           'E': lambda x: x.mode()
                         })
print(df2)

这里可以自定义list并加入进度条

agger = {
   'C': 'mean',
   'D': 'sum'}
   #'E': lambda x: x.mode()}
#加上'E'会报错:cannot perform both aggregation and transformation operations simultaneously
#这里的'E'为transformation opertions操作

gcols = ['B'] # columns defining the groups
groupby = df.groupby(gcols)

ngroups = len(groupby)
gfrac = 0.3 # fraction of groups for which you want to print progress
gfrac_size = max((1, int(ngroups*gfrac)))
#gfrac为比例,这里为0.3,ngroups为总的长度
#下面也就是将agg函数进行实现了一下
groups = []
rows = []
for i,g in tqdm(enumerate(groupby),total=len(groupby)):
    r"""
    i = 0,
    g = 
    (1.0,         A       B      C     D     E
      0          1.0     1.0    1.0   2.0    a
      1          2.0     1.0    1.5   5.0    a
      2          3.0     1.0    2.0   3.0    b)
    """
    gstats = g[1].agg(agger)
    r"""
    g[1] = 
         A    B    C    D   E
     0  1.0  1.0  1.0  2.0  a
     1  2.0  1.0  1.5  5.0  a
     2  3.0  1.0  2.0  3.0  b
    gstats = 
     C  1.5
     D  10.0
    newcols = 
    ['C', 'D']
    groups = 
    [1.0]
    rows = 
    [<numpy.flatiter object at 0x21d18a0>]
    
    g[1]取出了上面整体数据的前三个对应值
    g[1] = 
         A    B    C    D   E
     3  1.0  2.0  2.0  6.0  a
     4  2.0  2.0  3.0  4.0  b
     5  3.0  2.0  4.0  2.0  b
    gstats = 
     C     3.0
     D    12.0
    newcols = 
    ['C', 'D']
    groups = 
    [1.0, 2.0]
    rows = 
    [<numpy.flatiter object at 0x21d18a0>,<numpy.flatiter object at ox21cc610>,<numpy.flatiter object at 0x21cd060>]

	同理g[1]取出了上面的三个对应值
	g[1] = 
    	A    B     C    D  E
	6  1.0  3.0   5.0  5.0  b
	7  2.0  3.0   6.0  1.0  a
	8  3.0  3.0  10.0  2.0  a
	gstats = 
	C    7.0
	D    8.0
	newcols = 
	['C','D']
	groups = 
	[1.0,2.0,3.0]
	rows = 
	[<numpy.flatiter object at 0x21d18a0>, <numpy.flatiter object at 0x21cc610>, <numpy.flatiter object at 0x21cd060>]
    """
    if i==0:
        if gstats.ndim==2:
            newcols = gstats.columns.tolist()
        else:
            newcols = gstats.index.tolist()
    r"""
    newcols = ['C','D']
    newcols = ['C','D']
    newcols = ['C','D']
    """
    groups.append(g[0])
    r"""
    groups = [1.0]
    groups = [1.0,2.0]
    groups = [1.0,2.0,3.0]
    """
    rows.append(gstats.values.flat)
    r"""
    rows = [<numpy.flatiter object at 0x21d18a0>]
    rows = [<numpy.flatiter object at 0x21d18a0>, <numpy.flatiter object at 0x21cc610>]
    rows = [<numpy.flatiter object at 0x21d18a0>, <numpy.flatiter object at 0x21cc610>, <numpy.flatiter object at 0x21cd060>]
    """

df3 = pd.DataFrame(np.vstack(rows), columns=newcols)
if len(gcols) == 1:
    df3.index = groups
else:
    df3.index = pd.MultiIndex.from_tuples(groups, names=gcols)
df3 = df3.astype(df[newcols].dtypes)

之前每一次得到结果[1.5,10.0],[3.0,12.0]之类的，都是将数值展平之后压入rows

rows.append(gstats.values.flat)

因此这里得到新的结果之后，应该将新的结果进行展开得到对应的DataFrame

df3 = pd.DataFrame(np.vstack(rows),columns=newcols)

并且这里每次得到的列名0.0,1.0,2.0依次压入groups之中

groups.append(g[0])

因此这里最后对df3进行操作

if len(gcols) == 1:
	df3.index = groups
else:
	df3.index = pd.MultiIndex.from_tuples(groups,names=gcols)

这里将对应的group赋值到df3上面，得到结果

		C		D
1.0    1.5    10.0
2.0    3.0    12.0
3.0    7.0     8.0

方法2

import pandas as pd
import numpy as np
df = pd.DataFrame({"A":[1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0], 
                   "B":[1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0], 
                   "C":[1.0, 1.5, 2.0, 2.0, 3.0, 4.0, 5.0, 6.0, 10.0],
                   "D":[2.0, 5.0, 3.0, 6.0, 4.0, 2.0, 5.0, 1.0, 2.0],
                   "E":['a', 'a', 'b', 'a', 'b', 'b', 'b', 'a', 'a']}) 

df2 = df.groupby('B').agg({
                           'C': 'mean',
                           'D': 'sum',
                           'E': lambda x: x.mode()
                         })
print(df2)

class ModeHack:

    def __init__(self, size=5, N=10):
        self.ix = 0
        self.K = 1 
        self.size = size
        self.N = N

    def mode(self, x):
        self.ix = self.ix + x.shape[0]
        if self.K*self.size <= self.ix:
            print('{:.0f}% complete'.format(100*self.ix/self.N))
            self.K += 1

        return x.mode()

    def reset(self):    
        self.ix = 0
        self.K = 1

mymode = ModeHack(size=int(.1*df.shape[0]), N=df.shape[0])
mymode.reset()

agger = {
   'C': 'mean',
   'D': 'sum',
   'E': lambda x: mymode.mode(x)}

df3 = df.groupby('B').agg(agger)