DASK==python并行计算

文档10 Minutes to Dask — Dask documentation

demo代码

import numpy as np
import pandas as pd
import dask.dataframe as dd
import dask

# 设置调度器为多线程
dask.config.set(scheduler='threads')
# 创建一个示例的Pandas DataFrame
index = pd.date_range("2021-09-01", periods=3, freq="1h")
df = pd.DataFrame({"a": [1, 2, 5], "b": list("abc" * 1)}, index=index)
print(df)

# 将Pandas DataFrame转换为Dask DataFrame
ddf = dd.from_pandas(df, npartitions=2)
print('----------------')
print(ddf)
print('----------------')

# 执行一些计算
compute = ddf.a.mean()

# 打印Dask计算图
print(compute.dask)
print('----------------')

# 可视化Dask计算图
dask.visualize(compute, filename='dask_graph.png')

# 计算结果
compute_compute = compute.compute()
print(compute_compute)

执行计划和解析：

求平均

计算过程拆解

求方差

依赖包

pip list --format=freeze > requirements.txt
 

anyio==4.3.0
argon2-cffi==23.1.0
argon2-cffi-bindings==21.2.0
arrow==1.3.0
asttokens==2.4.1
async-lru==2.0.4
attrs==23.2.0
Babel==2.14.0
beautifulsoup4==4.12.3
bleach==6.1.0
bokeh==3.4.1
Brotli==1.1.0
cached-property==1.5.2
certifi==2024.2.2
cffi==1.16.0
charset-normalizer==3.3.2
click==8.1.7
cloudpickle==3.0.0
colorama==0.4.6
comm==0.2.2
contourpy==1.2.1
cytoolz==0.12.3
dask==2024.5.2
dask-expr==1.1.2
debugpy==1.8.1
decorator==5.1.1
defusedxml==0.7.1
distributed==2024.5.2
entrypoints==0.4
exceptiongroup==1.2.0
executing==2.0.1
fastjsonschema==2.19.1
fqdn==1.5.1
fsspec==2024.5.0
graphviz==0.20.3
h11==0.14.0
h2==4.1.0
hpack==4.0.0
httpcore==1.0.5
httpx==0.27.0
hyperframe==6.0.1
idna==3.7
importlib_metadata==7.1.0
importlib_resources==6.4.0
ipykernel==6.29.3
ipython==8.25.0
ipywidgets==8.1.3
isoduration==20.11.0
jedi==0.19.1
Jinja2==3.1.4
json5==0.9.25
jsonpointer==2.4
jsonschema==4.22.0
jsonschema-specifications==2023.12.1
jupyter==1.0.0
jupyter_client==8.6.2
jupyter-console==6.6.3
jupyter_core==5.7.2
jupyter-events==0.10.0
jupyter-lsp==2.2.5
jupyter_server==2.14.1
jupyter_server_terminals==0.5.3
jupyterlab==4.2.1
jupyterlab_pygments==0.3.0
jupyterlab_server==2.27.2
jupyterlab_widgets==3.0.11
locket==1.0.0
lz4==4.3.3
MarkupSafe==2.1.5
matplotlib-inline==0.1.7
mistune==3.0.2
msgpack==1.0.8
nbclient==0.10.0
nbconvert==7.16.4
nbformat==5.10.4
nest_asyncio==1.6.0
notebook==7.2.0
notebook_shim==0.2.4
numpy==1.26.4
overrides==7.7.0
packaging==24.0
pandas==2.2.2
pandocfilters==1.5.0
parso==0.8.4
partd==1.4.2
pickleshare==0.7.5
pillow==10.3.0
pip==24.0
pkgutil_resolve_name==1.3.10
platformdirs==4.2.2
prometheus_client==0.20.0
prompt-toolkit==3.0.42
psutil==5.9.8
pure-eval==0.2.2
pyarrow==16.1.0
pyarrow-hotfix==0.6
pycparser==2.22
Pygments==2.18.0
PySocks==1.7.1
python-dateutil==2.9.0
python-json-logger==2.0.7
pytz==2024.1
pywin32==306
pywinpty==2.0.13
PyYAML==6.0.1
pyzmq==26.0.3
qtconsole==5.5.2
QtPy==2.4.1
referencing==0.35.1
requests==2.32.3
rfc3339-validator==0.1.4
rfc3986-validator==0.1.1
rpds-py==0.18.1
Send2Trash==1.8.3
setuptools==70.0.0
six==1.16.0
sniffio==1.3.1
sortedcontainers==2.4.0
soupsieve==2.5
stack-data==0.6.2
tblib==3.0.0
terminado==0.18.1
tinycss2==1.3.0
tomli==2.0.1
toolz==0.12.1
tornado==6.4
traitlets==5.14.3
types-python-dateutil==2.9.0.20240316
typing_extensions==4.11.0
typing-utils==0.1.0
tzdata==2024.1
uri-template==1.3.0
urllib3==2.2.1
wcwidth==0.2.13
webcolors==1.13
webencodings==0.5.1
websocket-client==1.8.0
wheel==0.43.0
widgetsnbextension==4.0.11
win-inet-pton==1.1.0
xyzservices==2024.4.0
zict==3.0.0
zipp==3.17.0