Python进度条tqdm

简介

Python 和 CLI（命令行界面）的快速、可扩展的进度条

tqdm 源自阿拉伯语 taqaddum，意为“进步”，同时也是西班牙语“我爱你”的缩写 (te quiero demasiado)。

安装

pip install tqdm

初试

直接传入一个可迭代对象，如列表

from tqdm import tqdm
from time import sleep

text = ''
l = ['a', 'b', 'c', 'd']
for x in tqdm(l):
    sleep(0.25)
    text = text + x

主要用法

基于可迭代对象

直接传入一个可迭代对象，如列表

from tqdm import tqdm
from time import sleep

text = ''
l = ['a', 'b', 'c', 'd']
for x in tqdm(l):
    sleep(0.25)
    text = text + x

trange(i) 相当于 tqdm(range(i))

from time import sleep
from tqdm import trange

for i in trange(100):
    sleep(0.01)

允许外部实例化

from tqdm import tqdm
from time import sleep

pbar = tqdm(['a', 'b', 'c', 'd'])
for char in pbar:
    sleep(0.25)
    pbar.set_description('Processing %s' % char)

enumerate

tqdm() 应该被 enumerate() 包围，否则没有进度条

from tqdm import tqdm
from time import sleep

l = ['a', 'b', 'c', 'd']
for x in enumerate(tqdm(l)):
    sleep(0.25)

手动控制

使用 with 语句手动控制 tqdm() 更新

from tqdm import tqdm
from time import sleep

with tqdm(total=100) as pbar:
    for i in range(10):
        sleep(0.1)
        pbar.update(10)

像操作文件一样，tqdm() 可以赋值给一个变量，别忘了 del 或 close()

from tqdm import tqdm
from time import sleep

pbar = tqdm(total=100)
for i in range(10):
    sleep(0.1)
    pbar.update(10)
pbar.close()

自定义信息

设置描述：set_description()，显示在左边

设置后缀：set_postfix()，显示在右边

from time import sleep
from tqdm import tqdm, trange
from random import random, randint

with trange(10) as t:
    for i in t:
        t.set_description('GEN %i' % i)  # 描述显示在左边
        t.set_postfix(loss=random(), gen=randint(1, 999), str='h', lst=[1, 2])  # 后缀显示在右边，根据参数自动格式化
        sleep(0.1)

with tqdm(total=10, bar_format="{postfix[0]} {postfix[1][value]:>8.2g}", postfix=["Batch", dict(value=0)]) as t:
    for i in range(10):
        sleep(0.1)
        t.postfix[1]["value"] = i / 2
        t.update()

详细阅读：Description and additional stats

简略版

from tqdm import tqdm
from time import sleep

for x in tqdm(range(10), bar_format='正在执行 {l_bar}{bar}{r_bar}'):
    sleep(0.25)

嵌套进度条

from time import sleep
from tqdm.auto import trange

for i in trange(4, desc='1st loop'):
    for j in trange(5, desc='2nd loop'):
        for k in trange(50, desc='3rd loop', leave=False):
            sleep(0.01)

下载文件

import os
import urllib.request
from tqdm import tqdm


class TqdmUpTo(tqdm):
    def update_to(self, b=1, bsize=1, tsize=None):
        '''更新进度条

        :param b: 已转移的块数
        :param bsize: 每个块的大小
        :param tsize: 总大小
        :return:
        '''
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)


url = "https://caspersci.uk.to/matryoshka.zip"
with TqdmUpTo(unit='B', unit_scale=True, unit_divisor=1024, miniters=1, desc=os.path.basename(url)) as t:
    urllib.request.urlretrieve(url, filename=os.devnull, reporthook=t.update_to, data=None)
    t.total = t.n

详细阅读：Hooks and callbacks

集成

Pandas

调用 tqdm.pandas

import numpy as np
import pandas as pd
from tqdm import tqdm

df = pd.DataFrame(np.random.randint(0, 100, (100000, 6)))
tqdm.pandas(desc="my bar!")
df.progress_apply(lambda x: x ** 2)
df.groupby(0).progress_apply(lambda x: x ** 2)

Keras

from tqdm.keras import TqdmCallback

...

model.fit(..., verbose=0, callbacks=[TqdmCallback()])

实例

from keras.datasets import mnist
from keras.utils import np_utils
from keras.models import Sequential
from keras.layers.core import Dense
from tqdm.keras import TqdmCallback

(X, y), (_, _) = mnist.load_data()
X = X.reshape(len(X), -1)
X = X.astype('float32')
X = (X - 127) / 127
y = np_utils.to_categorical(y, num_classes=10)
model = Sequential()
model.add(Dense(20, input_shape=(784,), activation='relu'))
model.add(Dense(10, activation='sigmoid'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

model.fit(X, y, epochs=20, batch_size=64, verbose=0, callbacks=[TqdmCallback()])

IPython/Jupyter

from tqdm.notebook import trange, tqdm
from time import sleep

for i in trange(3, desc='1st loop'):
    for j in tqdm(range(100), desc='2nd loop'):
        sleep(0.01)

备注

1. 某些 IDE 如 PyCharm，一边输出一边更新，进度条将不连续。

显示在一行

PyCharm 有可能输出为多行，设置参数 dynamic_ncols=True

多进程多线程多协程

• 多进程 Process：multiprocessing
• 多线程 Thread：threading
• 多协程 Coroutine：asyncio

普通版

import time

from tqdm import tqdm


def f():
    time.sleep(0.01)


total = 1000
pbar = tqdm(total=total)
for i in range(total):
    f()
    pbar.update()

耗时 15.58 s

多进程

import time
from multiprocessing.pool import ThreadPool

from tqdm import tqdm


def f():
    time.sleep(0.01)


total = 1000
pbar = tqdm(total=total)  # 不能用上下文管理器
callback = lambda *args: pbar.update()
pool = ThreadPool(4)
for i in range(total):
    pool.apply_async(f, callback=callback)
pool.close()
pool.join()

耗时 3.89 s

进程池

import time
from concurrent.futures import ProcessPoolExecutor, as_completed

from tqdm import tqdm


def f():
    time.sleep(0.01)


if __name__ == '__main__':
    total = 1000
    pbar = tqdm(total=total)
    executor = ProcessPoolExecutor()
    futures = [executor.submit(f) for i in range(total)]
    for future in as_completed(futures):
        future.result()
        pbar.update()
    executor.shutdown()

耗时 5.25 s

多线程

import time
import threading

from tqdm import tqdm


def f():
    time.sleep(0.01)


if __name__ == '__main__':
    total = 1000
    pbar = tqdm(total=total)
    threads = []
    for i in range(total):
        threads.append(threading.Thread(target=f))
    for thread in threads:
        thread.start()
        pbar.update()
    for thread in threads:
        thread.join()

耗时 0.32 s

线程池

import time
from concurrent.futures import ThreadPoolExecutor, as_completed

from tqdm import tqdm


def f():
    time.sleep(0.01)


if __name__ == '__main__':
    total = 1000
    pbar = tqdm(total=total)
    executor = ThreadPoolExecutor()
    futures = [executor.submit(f) for i in range(total)]
    for future in as_completed(futures):
        future.result()
        pbar.update()
    executor.shutdown()

耗时 1.32 s

多协程

import asyncio

from tqdm import tqdm


async def f():
    await asyncio.sleep(0.01)


async def main():
    total = 1000
    pbar = tqdm(total=total)
    tasks = [f() for _ in range(total)]
    for task in asyncio.as_completed(tasks):
        await task
        pbar.update()


if __name__ == '__main__':
    asyncio.run(main())

耗时 0.32 s

参考文献

1. tqdm GitHub
2. tqdm 文档
3. MNIST & Keras保存模型并预测
4. python使用tqdm配置进度条及避免出现多行进度条
5. Python多线程处理数据，并打印进度条
6. concurrent.futures 多进程任务使用 tqdm 显示进度条
7. Use tqdm with concurrent.futures?
8. Making a tqdm progress bar for asyncio