基于 miniconda 构建包含 torch dgl 等包的python环境

probtions

于 2024-03-23 16:38:06 发布

阅读量724

点赞数 24

分类专栏： gpu 文章标签： python pytorch tensorflow

本文链接：https://blog.csdn.net/weixin_46028840/article/details/136969646

版权

gpu 专栏收录该内容

1 篇文章 0 订阅

订阅专栏

wget https://repo.anaconda.com/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh
./Miniconda3-4.5.4-Linux-x86_64.sh -b -p venv
source venv/bin/activate
pip install --upgrade pip

第1部分

pip install typing_extensions
pip download torch==1.9.1+cu111 torchvision==0.10.1+cu111 torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html
pip install torch==1.9.1+cu111 torchvision==0.10.1+cu111 torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html
pip install https://data.dgl.ai/wheels/dgl_cu111-0.7.1-cp36-cp36m-manylinux1_x86_64.whl

(base) [root@docker venv]# pip download torch==1.9.1+cu111 torchvision==0.10.1+cu111 torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html
Looking in links: https://download.pytorch.org/whl/torch_stable.html
Collecting torch==1.9.1+cu111
  Using cached https://download.pytorch.org/whl/cu111/torch-1.9.1%2Bcu111-cp36-cp36m-linux_x86_64.whl (2041.3 MB)
Collecting torchvision==0.10.1+cu111
  Using cached https://download.pytorch.org/whl/cu111/torchvision-0.10.1%2Bcu111-cp36-cp36m-linux_x86_64.whl (20.6 MB)
Collecting torchaudio==0.9.1
  Using cached torchaudio-0.9.1-cp36-cp36m-manylinux1_x86_64.whl (1.9 MB)
Collecting typing-extensions
  Using cached https://download.pytorch.org/whl/typing_extensions-4.8.0-py3-none-any.whl (31 kB)
Collecting dataclasses
  Downloading dataclasses-0.8-py3-none-any.whl (19 kB)
Collecting pillow>=5.3.0
  Downloading Pillow-8.4.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)
     |████████████████████████████████| 3.1 MB 76 kB/s             
Collecting numpy
  Downloading numpy-1.19.5-cp36-cp36m-manylinux2010_x86_64.whl (14.8 MB)
     |████████████████████████████████| 14.8 MB 77 kB/s             
Collecting typing-extensions
  Downloading https://download.pytorch.org/whl/typing_extensions-4.4.0-py3-none-any.whl (26 kB)
  Using cached typing_extensions-4.1.1-py3-none-any.whl (26 kB)
Saved ./torch-1.9.1+cu111-cp36-cp36m-linux_x86_64.whl
Saved ./torchvision-0.10.1+cu111-cp36-cp36m-linux_x86_64.whl
Saved ./torchaudio-0.9.1-cp36-cp36m-manylinux1_x86_64.whl
Saved ./Pillow-8.4.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
Saved ./dataclasses-0.8-py3-none-any.whl
Saved ./numpy-1.19.5-cp36-cp36m-manylinux2010_x86_64.whl
Saved ./typing_extensions-4.1.1-py3-none-any.whl
Successfully downloaded torch torchvision torchaudio pillow dataclasses numpy typing-extensions


(base) [root@docker venv]# pip install torch==1.9.1+cu111 torchvision==0.10.1+cu111 torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html
Looking in links: https://download.pytorch.org/whl/torch_stable.html
Collecting torch==1.9.1+cu111
  Using cached https://download.pytorch.org/whl/cu111/torch-1.9.1%2Bcu111-cp36-cp36m-linux_x86_64.whl (2041.3 MB)
Collecting torchvision==0.10.1+cu111
  Using cached https://download.pytorch.org/whl/cu111/torchvision-0.10.1%2Bcu111-cp36-cp36m-linux_x86_64.whl (20.6 MB)
Collecting torchaudio==0.9.1
  Using cached torchaudio-0.9.1-cp36-cp36m-manylinux1_x86_64.whl (1.9 MB)
Collecting dataclasses
  Using cached dataclasses-0.8-py3-none-any.whl (19 kB)
Requirement already satisfied: typing-extensions in ./venv/lib/python3.6/site-packages (from torch==1.9.1+cu111) (4.1.1)
Collecting pillow>=5.3.0
  Using cached Pillow-8.4.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)
Collecting numpy
  Using cached numpy-1.19.5-cp36-cp36m-manylinux2010_x86_64.whl (14.8 MB)
Installing collected packages: dataclasses, torch, pillow, numpy, torchvision, torchaudio
Successfully installed dataclasses-0.8 numpy-1.19.5 pillow-8.4.0 torch-1.9.1+cu111 torchaudio-0.9.1 torchvision-0.10.1+cu111
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv


(base) [root@docker venv]# pip list
Package           Version
----------------- ------------
asn1crypto        0.24.0
certifi           2018.4.16
cffi              1.11.5
chardet           3.0.4
conda             4.5.4
cryptography      2.2.2
dataclasses       0.8
idna              2.6
numpy             1.19.5
Pillow            8.4.0
pip               21.3.1
pycosat           0.6.3
pycparser         2.18
pyOpenSSL         18.0.0
PySocks           1.6.8
requests          2.18.4
ruamel_yaml       0.15.37
setuptools        39.2.0
six               1.11.0
torch             1.9.1+cu111
torchaudio        0.9.1
torchvision       0.10.1+cu111
typing_extensions 4.1.1
urllib3           1.22
wheel             0.31.1


(base) [root@docker venv]#
(base) [root@docker venv]# pip install https://data.dgl.ai/wheels/dgl_cu111-0.7.1-cp36-cp36m-manylinux1_x86_64.whl
Collecting dgl-cu111==0.7.1
  Downloading https://data.dgl.ai/wheels/dgl_cu111-0.7.1-cp36-cp36m-manylinux1_x86_64.whl (164.8 MB)
     |████████████████████████████████| 164.8 MB 25 kB/s             
Collecting requests>=2.19.0
  Downloading requests-2.27.1-py2.py3-none-any.whl (63 kB)
     |████████████████████████████████| 63 kB 417 kB/s            
Collecting scipy>=1.1.0
  Downloading scipy-1.5.4-cp36-cp36m-manylinux1_x86_64.whl (25.9 MB)
     |████████████████████████████████| 25.9 MB 69 kB/s             
Collecting networkx>=2.1
  Downloading networkx-2.5.1-py3-none-any.whl (1.6 MB)
     |████████████████████████████████| 1.6 MB 48 kB/s             
Requirement already satisfied: numpy>=1.14.0 in ./venv/lib/python3.6/site-packages (from dgl-cu111==0.7.1) (1.19.5)
Collecting decorator<5,>=4.3
  Downloading decorator-4.4.2-py2.py3-none-any.whl (9.2 kB)
Collecting charset-normalizer~=2.0.0
  Downloading charset_normalizer-2.0.12-py3-none-any.whl (39 kB)
Requirement already satisfied: certifi>=2017.4.17 in ./venv/lib/python3.6/site-packages (from requests>=2.19.0->dgl-cu111==0.7.1) (2018.4.16)
Requirement already satisfied: idna<4,>=2.5 in ./venv/lib/python3.6/site-packages (from requests>=2.19.0->dgl-cu111==0.7.1) (2.6)
Requirement already satisfied: urllib3<1.27,>=1.21.1 in ./venv/lib/python3.6/site-packages (from requests>=2.19.0->dgl-cu111==0.7.1) (1.22)
Installing collected packages: decorator, charset-normalizer, scipy, requests, networkx, dgl-cu111
  Attempting uninstall: requests
    Found existing installation: requests 2.18.4
    Uninstalling requests-2.18.4:
      Successfully uninstalled requests-2.18.4
Successfully installed charset-normalizer-2.0.12 decorator-4.4.2 dgl-cu111-0.7.1 networkx-2.5.1 requests-2.27.1 scipy-1.5.4
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
(base) [root@docker venv]#

第2部分

jupyter
pandas
transformers
arrow
xlrd
fastjsonschema
gitdb

pip install jupyter -i https://pypi.tuna.tsinghua.edu.cn/simple
Installing collected packages: zipp, ipython-genutils, traitlets, pyrsistent, importlib-metadata, attrs, wcwidth, tornado, pyzmq, python-dateutil, pyparsing, ptyprocess, parso, nest-asyncio, jupyter-core, jsonschema, entrypoints, webencodings, pygments, prompt-toolkit, pickleshare, pexpect, packaging, nbformat, MarkupSafe, jupyter-client, jedi, backcall, async-generator, testpath, pandocfilters, nbclient, mistune, jupyterlab-pygments, jinja2, ipython, defusedxml, bleach, argon2-cffi-bindings, terminado, Send2Trash, prometheus-client, nbconvert, ipykernel, argon2-cffi, notebook, widgetsnbextension, qtpy, jupyterlab-widgets, comm, qtconsole, jupyter-console, ipywidgets, jupyter
Successfully installed MarkupSafe-2.0.1 Send2Trash-1.8.2 argon2-cffi-21.3.0 argon2-cffi-bindings-21.2.0 async-generator-1.10 attrs-22.2.0 backcall-0.2.0 bleach-4.1.0 comm-0.1.4 defusedxml-0.7.1 entrypoints-0.4 importlib-metadata-4.8.3 ipykernel-5.5.6 ipython-7.16.3 ipython-genutils-0.2.0 ipywidgets-7.8.1 jedi-0.17.2 jinja2-3.0.3 jsonschema-3.2.0 jupyter-1.0.0 jupyter-client-7.1.2 jupyter-console-6.4.3 jupyter-core-4.9.2 jupyterlab-pygments-0.1.2 jupyterlab-widgets-1.1.7 mistune-0.8.4 nbclient-0.5.9 nbconvert-6.0.7 nbformat-5.1.3 nest-asyncio-1.6.0 notebook-6.4.10 packaging-21.3 pandocfilters-1.5.1 parso-0.7.1 pexpect-4.9.0 pickleshare-0.7.5 prometheus-client-0.17.1 prompt-toolkit-3.0.36 ptyprocess-0.7.0 pygments-2.14.0 pyparsing-3.0.7 pyrsistent-0.18.0 python-dateutil-2.9.0.post0 pyzmq-25.1.2 qtconsole-5.2.2 qtpy-2.0.1 terminado-0.12.1 testpath-0.6.0 tornado-6.1 traitlets-4.3.3 wcwidth-0.2.13 webencodings-0.5.1 widgetsnbextension-3.6.6 zipp-3.6.0


pip install pandas -i https://pypi.tuna.tsinghua.edu.cn/simple
Installing collected packages: pytz, pandas
Successfully installed pandas-1.1.5 pytz-2024.1


pip install transformers -i https://pypi.tuna.tsinghua.edu.cn/simple

Installing collected packages: importlib-resources, tqdm, regex, pyyaml, joblib, filelock, click, tokenizers, sacremoses, huggingface-hub, transformers
Successfully installed click-8.0.4 filelock-3.4.1 huggingface-hub-0.4.0 importlib-resources-5.4.0 joblib-1.1.1 pyyaml-6.0.1 regex-2023.8.8 sacremoses-0.0.53 tokenizers-0.12.1 tqdm-4.64.1 transformers-4.18.0


pip install arrow -i https://pypi.tuna.tsinghua.edu.cn/simple
Installing collected packages: arrow
Successfully installed arrow-1.2.3

pip install xlrd -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed xlrd-2.0.1


pip install fastjsonschema -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed fastjsonschema-2.19.1

pip install gitdb -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed gitdb-4.0.9 smmap-5.0.0

第3部分

sentencepiece
sniffio
stack-data
termcolor
tzdata
yarl
wrapt

pip install sentencepiece -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed sentencepiece-0.2.0

pip install sniffio -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed contextvars-2.4 immutables-0.19 sniffio-1.2.0

pip install stack-data -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed asttokens-2.4.1 executing-2.0.1 pure-eval-0.2.2 six-1.16.0 stack-data-0.6.3

pip install termcolor -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed termcolor-1.1.0

pip install tzdata -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed tzdata-2024.1

pip install yarl -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed multidict-5.2.0 yarl-1.7.2


pip install wrapt -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed wrapt-1.16.0

第4部分

tinycss2
soupsieve
sentence-transformers
rfc3339-validator
rfc3986-validator
pytorch-lightning
python-json-logger
opt-einsum
notebook-shim
nbclassic
matplotlib
matplotlib-inline

pip install tinycss2 soupsieve  -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed soupsieve-2.3.2.post1 tinycss2-1.1.1


pip install sentence-transformers -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed nltk-3.6.7 sentence-transformers-2.2.2


pip install rfc3339-validator rfc3986-validator  -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed rfc3339-validator-0.1.4 rfc3986-validator-0.1.1


pip install pytorch-lightning python-json-logger -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed aiohttp-3.8.6 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 frozenlist-1.2.0 fsspec-2022.1.0 future-1.0.0 idna-ssl-1.1.0 pyDeprecate-0.3.1 python-json-logger-2.0.7 pytorch-lightning-1.5.10 setuptools-59.5.0


pip install opt-einsum notebook-shim nbclassic -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed opt-einsum-3.3.0
Successfully installed anyio-3.6.2 idna-3.6 jupyter-server-1.13.1 notebook-shim-0.0.5
Successfully installed nbclassic-0.3.5

pip install matplotlib matplotlib-inline -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed cycler-0.11.0 kiwisolver-1.3.1 matplotlib-3.3.4
Successfully installed matplotlib-inline-0.1.6

第5部分

tensorboardX
protobuf
tensorflow-estimator
keras-nightly
Keras-Preprocessing
astunparse
beautifulsoup4
debugpy
flatbuffers
fonttools
fqdn
gast
google-pasta
h5py
jsonpointer
InstructorEmbedding

pip install tensorboardX  -i https://pypi.tuna.tsinghua.edu.cn/simple
pip install tensorboardX  protobuf==3.19.6 -i https://pypi.tuna.tsinghua.edu.cn/simple     注：protobuf==3.19.6 指定具体版本，解决冲突
Successfully installed protobuf-3.19.6 tensorboardX-2.6


pip install tensorflow-estimator  -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed tensorflow-estimator-2.8.0


pip install keras-nightly Keras-Preprocessing -i https://pypi.tuna.tsinghua.edu.cn/simple
pip install keras-nightly Keras-Preprocessing okok
Successfully installed Keras-Preprocessing-1.1.2 keras-nightly-2.11.0.dev2022092207


pip install astunparse beautifulsoup4 debugpy flatbuffers fonttools fqdn gast google-pasta h5py jsonpointer -i https://pypi.tuna.tsinghua.edu.cn/simple
Installing collected packages: cached-property, jsonpointer, h5py, google-pasta, gast, fqdn, fonttools, flatbuffers, debugpy, beautifulsoup4, astunparse
Successfully installed astunparse-1.6.3 beautifulsoup4-4.12.3 cached-property-1.5.2 debugpy-1.5.1 flatbuffers-24.3.7 fonttools-4.27.1 fqdn-1.5.1 gast-0.5.4 google-pasta-0.2.0 h5py-3.1.0 jsonpointer-2.3


pip install InstructorEmbedding -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed InstructorEmbedding-1.0.1

第6部分

jupyter-events
jupyter_server_terminals
contourpy
isoduration
lightning-utilities
tensorflow-gpu
safetensors

pip install jupyter-events -i https://pypi.tuna.tsinghua.edu.cn/simple
pip install jupyter_server_terminals -i https://pypi.tuna.tsinghua.edu.cn/simple

ERROR: Could not find a version that satisfies the requirement jupyter-events (from versions: none)
ERROR: Could not find a version that satisfies the requirement jupyter_server_terminals (from versions: none)

问题分析：
jupyter-events==0.6.3    Released: Jan 13, 2023    Requires: Python >=3.7
jupyter-events==0.1.0    Released: May 31, 2022    Requires: Python >=3.7
jupyter-server-terminals==0.4.4    Released: Jan 9, 2023    Requires: Python >=3.8
jupyter-server-terminals==0.1.0    Released: Apr 3, 2022    Requires: Python >=3.7


pip install contourpy isoduration

ERROR: Could not find a version that satisfies the requirement contourpy (from versions: none)
ERROR: Could not find a version that satisfies the requirement isoduration (from versions: none)
问题分析：
contourpy==0.0.1    Released: Sep 21, 2021    Requires: Python >=3.7
isoduration==20.11.0    Released: Nov 1, 2020    Requires: Python >=3.7



pip install lightning-utilities
ERROR: Could not find a version that satisfies the requirement lightning-utilities (from versions: none)
ERROR: No matching distribution found for lightning-utilities
问题分析：
lightning-utilities==0.9.0    Released: Jun 30, 2023   Requires: Python >=3.7
lightning-utilities==0.2.0    Released: Sep 6, 2022    Requires: Python >=3.7



pip install tensorflow-gpu  -i https://pypi.tuna.tsinghua.edu.cn/simple

报错：
    =========================================================
    The "tensorflow-gpu" package has been removed!
    
    Please install "tensorflow" instead.
    
    Other than the name, the two packages have been identical
    since TensorFlow 2.1, or roughly since Sep 2019. For more
    information, see: pypi.org/project/tensorflow-gpu
    =========================================================



pip install safetensors  -i https://pypi.tuna.tsinghua.edu.cn/simple
报错：
error: can't find Rust compiler  暂未解决

safetensors==0.4.0   Requires: Python >=3.7

第7部分

torchmetrics
tensorboard
torch-geometric
websocket-client
webcolors
wandb
uri-template

pip install torchmetrics
Installing collected packages: pyDeprecate, torchmetrics
Successfully installed pyDeprecate-0.3.2 torchmetrics-0.8.2


pip install tensorboard -i https://pypi.tuna.tsinghua.edu.cn/simple
Installing collected packages: pyasn1, rsa, pyasn1-modules, oauthlib, cachetools, requests-oauthlib, google-auth, werkzeug, tensorboard-plugin-wit, tensorboard-data-server, setuptools, protobuf, markdown, grpcio, google-auth-oauthlib, absl-py, tensorboard
  Attempting uninstall: setuptools
    Found existing installation: setuptools 39.2.0
    Uninstalling setuptools-39.2.0:
      Successfully uninstalled setuptools-39.2.0
Successfully installed absl-py-1.4.0 cachetools-4.2.4 google-auth-2.22.0 google-auth-oauthlib-0.4.6 grpcio-1.48.2 markdown-3.3.7 oauthlib-3.2.2 protobuf-3.19.6 pyasn1-0.5.1 pyasn1-modules-0.3.0 requests-oauthlib-1.4.0 rsa-4.9 setuptools-59.6.0 tensorboard-2.10.1 tensorboard-data-server-0.6.1 tensorboard-plugin-wit-1.8.1 werkzeug-2.0.3


pip install torch-geometric -i https://pypi.tuna.tsinghua.edu.cn/simple
Installing collected packages: threadpoolctl, isodate, yacs, scikit-learn, rdflib, googledrivedownloader, torch-geometric
Successfully installed googledrivedownloader-0.4 isodate-0.6.1 rdflib-5.0.0 scikit-learn-0.24.2 threadpoolctl-3.1.0 torch-geometric-2.0.3 yacs-0.1.8


pip install websocket-client webcolors wandb uri-template -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed GitPython-3.1.18 appdirs-1.4.4 docker-pycreds-0.4.0 pathtools-0.1.2 psutil-5.9.8 sentry-sdk-1.43.0 setproctitle-1.2.3 uri-template-1.2.0 urllib3-1.26.18 wandb-0.15.12 webcolors-1.11.1 websocket-client-1.3.1

第8部分

torch-sparse
torch-scatter

pip install torch-sparse -i https://pypi.tuna.tsinghua.edu.cn/simple
Successfully installed torch-sparse-0.6.12

注：torch_sparse-0.6.12.tar.gz torch-sparse 安装需要一定时间



pip install torch-scatter -i https://pypi.tuna.tsinghua.edu.cn/simple

报错：
subprocess.CalledProcessError: Command '['which', 'g++-7']' returned non-zero exit status 1.


(base) [root@docker venv]# pip install torch-scatter
Collecting torch-scatter
  Using cached torch_scatter-2.0.9.tar.gz (21 kB)
  Preparing metadata (setup.py) ... done
Building wheels for collected packages: torch-scatter
  Building wheel for torch-scatter (setup.py) ... done
  Created wheel for torch-scatter: filename=torch_scatter-2.0.9-cp36-cp36m-linux_x86_64.whl size=279609 sha256=a05d2afc6b3919d03f19c1f3688c27ba6e56495ad18ac8f8820c8ea1efe97eb5
  Stored in directory: /root/.cache/pip/wheels/48/f1/07/fe02c3b61f43390870ff365ee2a1d690c2becf263ab3328273
Successfully built torch-scatter
Installing collected packages: torch-scatter
Successfully installed torch-scatter-2.0.9
WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv
(base) [root@docker venv]#

已解决：
centOS默认自带4点几版本的gcc 、g++编译器，是比较老的版本的 gcc version 4.8.5

# yum install centos-release-scl
安装 devtoolset-7，这个包含g++ 7的软件集：
# yum install devtoolset-7
启用 devtoolset-7 环境
# scl enable devtoolset-7 bash
验证g++版本
# g++ --version
g++ (GCC) 7.3.1 20180303 (Red Hat 7.3.1-5)
Copyright (C) 2017 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

# which g++
/opt/rh/devtoolset-7/root/usr/bin/g++

# cp /opt/rh/devtoolset-7/root/usr/bin/g++ /opt/rh/devtoolset-7/root/usr/bin/g++-7

# which g++-7
/opt/rh/devtoolset-7/root/usr/bin/g++-7

在CentOS系统上安装g++-7，你可以使用Software Collections (SCL) repository。

注：

上述命令会启动一个新的shell，在这个shell中，g++的版本将会是7.x.x。如果你想要在任何新的shell中都保持这个版本，你可以在启用devtoolset-7后将g++的符号链接到正确的版本
ln -s /opt/rh/devtoolset-7/root/usr/bin/g++ /usr/bin/g++